local api (#482)

Co-authored-by: AlteredCoder
Co-authored-by: erenJag

.github/workflows/basic_functionnals_tests.yml

@@ -1,86 +0,0 @@
-name: Basic functionals tests
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
-jobs:
-  build:
-    name: Install generated release and perform basic tests
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.13
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.13
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-    - name: Build release
-      run: BUILD_VERSION=xxx make release
-    # - name: Cache release directory
-    #   uses: actions/cache@v2
-    #   with:
-    #     path: ./crowdsec-xxx
-    #     key: ${{ runner.os }}-crowdsec-xxx
-    - name: Install release
-      run: |
-        cd crowdsec-xxx
-        sudo ./wizard.sh --bininstall
-        sudo cscli update
-        sudo sed -i 's/api: true/api: false/g' /etc/crowdsec/config/default.yaml
-    - name: Install collection
-      run: |
-        sudo cscli list -a
-        sudo cscli install parser crowdsecurity/syslog-logs crowdsecurity/sshd-logs crowdsecurity/dateparse-enrich
-        sudo cscli install scenario crowdsecurity/ssh-bf
-    - name: Crowdsec Startup check
-      run: |
-        sudo crowdsec -c /etc/crowdsec/config/user.yaml -t 
-    - name: Generate fake ssh bf logs
-      run: |
-        for i in `seq 1 10` ; do 
-          echo `date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
-        done;
-    - name: Process ssh-bf logs in time-machine
-      run: |
-        sudo crowdsec -c /etc/crowdsec/config/user.yaml -file ./ssh-bf.log -type syslog
-    - name: Cscli ban list check
-      run: |
-        sudo cscli ban list
-        sudo cscli ban list -o json | jq -e '.[].iptext == "1.1.1.172"'
-        sudo cscli ban list --range 1.1.1.0/24 -o json | jq -e '.[].iptext == "1.1.1.172"'
-    - name: Cscli ban del check
-      run: |
-        sudo cscli ban del ip 1.1.1.172
-        sudo cscli -c /etc/crowdsec/config/user.yaml ban list -o json | jq -e '. == null'
-    - name: Service start
-      run: |
-        sudo rm -f /etc/crowdsec/config/acquis.yaml 
-        touch /tmp/test.log
-        echo "filename: /tmp/test.log" | sudo tee -a /etc/crowdsec/config/acquis.yaml > /dev/null
-        echo "labels:" | sudo tee -a /etc/crowdsec/config/acquis.yaml > /dev/null
-        echo "  type: syslog" | sudo tee -a /etc/crowdsec/config/acquis.yaml > /dev/null
-        sudo systemctl restart crowdsec
-    - name: Service status check
-      run: |
-        sleep 3
-        sudo cat /var/log/crowdsec.log
-        sudo systemctl status crowdsec
-        sudo cscli metrics
-    - name: Inject logs
-      run: |
-        cat ssh-bf.log >> /tmp/test.log
-        sleep 1
-    - name: Check results
-      run: |
-        sudo cscli ban list
-        sudo cscli ban list -o json | jq -e '.[].iptext == "1.1.1.172"'
-        sudo cat /var/log/crowdsec.log
-    - name: Check metrics
-      run: |
-        sudo cscli metrics
-          
-

.github/workflows/ci_functests-install.yml

@@ -0,0 +1,90 @@
+name: Hub-CI
+
+on:
+  push:
+    branches:
+      - wip_lapi
+      - master
+  pull_request:
+    branches:
+      - wip_lapi
+      - master
+
+jobs:
+  build:
+    name: Install generated release and perform basic tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.13
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.13
+      id: go
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Build release
+      run: BUILD_VERSION=xxx make release
+    - name: Install release
+      run: |
+        cd crowdsec-xxx
+        sudo bash -x ./wizard.sh --bininstall
+        sudo cscli machines add -a
+    - name: Post-installation check
+      run: |
+        sudo cscli hub update
+    - name: Install collection
+      run: |
+        sudo cscli hub list -a
+        sudo cscli parsers install crowdsecurity/syslog-logs crowdsecurity/sshd-logs crowdsecurity/dateparse-enrich
+        sudo cscli scenarios install crowdsecurity/ssh-bf
+    - name: Crowdsec start service
+      run: |
+        sudo systemctl start crowdsec
+    - name: Generate fake ssh bf logs
+      run: |
+        for i in `seq 1 10` ; do 
+          echo `date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
+        done;
+    - name: Process ssh-bf logs in time-machine
+      run: |
+        sudo crowdsec -file ./ssh-bf.log -type syslog -no-api
+    - name: Cscli ban list check
+      #check that we got the expected ban and that the filters are working properly
+      run: |
+        sudo cscli decisions list
+        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions list -r 1.1.1.0/24 -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions list -r 1.1.2.0/24 -o=json | jq -e '. == null'
+        sudo cscli decisions list -i 1.1.1.172 -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions list -i 1.1.1.173 -o=json | jq -e '. == null'
+    - name: Cscli ban del check
+      #check that the delete is working and that filters are working properly
+      run: |
+        sudo cscli decisions delete -i 1.1.1.173
+        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+        sudo cscli decisions delete -i 1.1.1.172
+        sudo cscli decisions list -o=json | jq -e '. == null'
+    - name: Metrics check
+      run: |
+        sudo cscli metrics
+    - name: Service stop & config change
+      #shutdown the service, edit that acquisition.yaml
+      run: |
+        sudo systemctl stop crowdsec
+        echo "" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        echo "filename: /tmp/test.log" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        echo "labels:" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        echo "  type: syslog" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
+        touch /tmp/test.log
+    - name: Service start & check
+      run: |
+        sudo systemctl start crowdsec || sudo journalctl -xe
+    - name: Trigger events via normal acquisition
+      run: |
+        cat ssh-bf.log >> /tmp/test.log
+        sleep 1
+    - name: Check results
+      run: |
+        sudo cscli decisions list -o=json | jq -e '.[].decisions[0].value == "1.1.1.172"'
+
+

.github/workflows/ci_go-test.yml

@@ -27,9 +27,9 @@ jobs:
       uses: jandelgado/gcov2lcov-action@v1.0.2
       with:
         infile: coverage.out
-        outfile: coverage.lcov
+        outfile: coverage.txt
     - name: Coveralls
       uses: coverallsapp/github-action@master
       with:
         github-token: ${{ secrets.GITHUB_TOKEN }}
-        path-to-lcov: coverage.lcov
+        path-to-lcov: coverage.txt

.github/workflows/ci_golangci-lint.yml

@@ -20,3 +20,5 @@ jobs:
           # Optional: golangci-lint command line arguments.
           args: --issues-exit-code=0 --timeout 5m
           only-new-issues: true
+
+

.github/workflows/hub_tests.yml → .github/workflows/ci_hub-tests.yml

@@ -1,4 +1,4 @@
-name: Hub tests
+name: Hub-CI
 
 on:
   push:
@@ -33,27 +33,26 @@ jobs:
         git clone https://github.com/crowdsecurity/hub-tests.git
         cd hub-tests
         make
-    - id: keydb
-      uses: pozetroninc/github-action-get-latest-release@master
+    - uses: oprypin/find-latest-tag@v1
       with:
-        owner: crowdsecurity
-        repo: crowdsec
-        excludes: prerelease, draft
+        repository: crowdsecurity/crowdsec  # The repository to scan.
+        releases-only: false  # We know that all relevant tags have a GitHub release for them.
+      id: crowdsec  # The step ID to refer to later.
     - name: Create crowdsec test env with all parsers from the release
       run: |
-        cd crowdsec-${{ steps.keydb.outputs.release }}
+        cd crowdsec-${{ steps.crowdsec.outputs.tag }}
         ./test_env.sh
         cd tests
         for i in `./cscli -c dev.yaml list parsers -a -o json | jq -r ".[].name" ` ; do 
           ./cscli -c dev.yaml install parser $i ; 
         done
     - name: Setup hub ci in crowdsec
-      working-directory: ./crowdsec-${{ steps.keydb.outputs.release }}/tests/
+      working-directory: ./crowdsec-${{ steps.crowdsec.outputs.tag }}/tests/
       run: |
         cp -R ../../hub-tests/tests .
         cp ../../hub-tests/main .
     - name: Run the HUB CI
-      working-directory: ./crowdsec-${{ steps.keydb.outputs.release }}/tests/
+      working-directory: ./crowdsec-${{ steps.crowdsec.outputs.tag }}/tests/
       run: |
         for i in `find ./tests -mindepth 1 -maxdepth 1 -type d` ; do 
           echo "::group::Test-${i}" ; 

Dockerfile

@@ -0,0 +1,22 @@
+ARG GOVERSION=1.14
+
+FROM golang:${GOVERSION}-alpine AS build
+
+WORKDIR /go/src/crowdsec
+COPY . .
+
+RUN apk update && apk add git jq gcc libc-dev make bash gettext
+RUN BUILD_VERSION="$(git describe --tags `git rev-list --tags --max-count=1`)" make release
+RUN /bin/bash wizard.sh --docker-mode
+RUN cscli hub update && cscli collections install crowdsecurity/linux
+
+FROM alpine:latest
+COPY --from=build /etc/crowdsec /etc/crowdsec
+COPY --from=build /var/lib/crowdsec /var/lib/crowdsec
+COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
+COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
+COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
+COPY --from=build /go/src/crowdsec/docker/config.yaml /etc/crowdsec/config.yaml
+
+
+ENTRYPOINT /bin/sh docker_start.sh

Makefile

@@ -3,7 +3,6 @@ CFG_PREFIX = $(PREFIX)"/etc/crowdsec/"
 BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
 DATA_PREFIX = $(PREFIX)"/var/run/crowdsec/"
 
-PLUGIN_FOLDER="./plugins"
 PID_DIR = $(PREFIX)"/var/run/"
 CROWDSEC_FOLDER = "./cmd/crowdsec"
 CSCLI_FOLDER = "./cmd/crowdsec-cli/"
@@ -14,14 +13,13 @@ BUILD_CMD="build"
 GOARCH=amd64
 GOOS=linux
 
-
-#Current versioning information from env
+#Golang version info
 GO_MAJOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1)
 GO_MINOR_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f2)
 MINIMUM_SUPPORTED_GO_MAJOR_VERSION = 1
 MINIMUM_SUPPORTED_GO_MINOR_VERSION = 13
 GO_VERSION_VALIDATION_ERR_MSG = Golang version ($(BUILD_GOVERSION)) is not supported, please use least $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION).$(MINIMUM_SUPPORTED_GO_MINOR_VERSION)
-
+#Current versioning information from env
 BUILD_VERSION?="$(shell git describe --tags `git rev-list --tags --max-count=1`)"
 BUILD_GOVERSION="$(shell go version | cut -d " " -f3 | sed -r 's/[go]+//g')"
 BUILD_CODENAME=$(shell cat RELEASE.json | jq -r .CodeName)
@@ -36,20 +34,20 @@ RELDIR = crowdsec-$(BUILD_VERSION)
 
 all: clean test build
 
-build: clean goversion crowdsec cscli
+build: goversion crowdsec cscli
 
 static: goversion crowdsec_static cscli_static
 
 goversion:
 	@if [ $(GO_MAJOR_VERSION) -gt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
-		exit 0 ;\
-	elif [ $(GO_MAJOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
-		echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
-		exit 1; \
-	elif [ $(GO_MINOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MINOR_VERSION) ] ; then \
-		echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
-		exit 1; \
-	fi
+        exit 0 ;\
+    elif [ $(GO_MAJOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MAJOR_VERSION) ]; then \
+        echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
+        exit 1; \
+    elif [ $(GO_MINOR_VERSION) -lt $(MINIMUM_SUPPORTED_GO_MINOR_VERSION) ] ; then \
+        echo '$(GO_VERSION_VALIDATION_ERR_MSG)';\
+        exit 1; \
+    fi
 
 hubci:
 	@rm -rf crowdsec-xxx hub-tests
@@ -69,26 +67,50 @@ clean:
 	@rm -f $(CSCLI_BIN)
 	@rm -f *.log
 
-cscli: goversion
+cscli:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CSCLI_FOLDER) build --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 
-
-crowdsec: goversion
+crowdsec:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CROWDSEC_FOLDER) build --no-print-directory
-	@bash ./scripts/build_plugins.sh
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 
-cscli_static: goversion
+cscli_static:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CSCLI_FOLDER) static --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 
-crowdsec_static: goversion
+crowdsec_static:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CROWDSEC_FOLDER) static --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
+
 
 #.PHONY: test
 test:
+ifeq ($(lastword $(RESPECT_VERSION)), $(CURRENT_GOVERSION))
 	@make -C $(CROWDSEC_FOLDER) test --no-print-directory
+else
+	@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
+	@exit 1;
+endif
 
 .PHONY: uninstall
 uninstall:
@@ -110,7 +132,4 @@ release: check_release build
 	@cp -R ./config/ $(RELDIR)
 	@cp wizard.sh $(RELDIR)
 	@cp scripts/test_env.sh $(RELDIR)
-	@bash ./scripts/build_plugins.sh
-	@mkdir -p "$(RELDIR)/plugins/backend"
-	@find ./plugins -type f -name "*.so" -exec install -Dm 644 {} "$(RELDIR)/{}" \; || exiting 
 	@tar cvzf crowdsec-release.tgz $(RELDIR)	

cmd/crowdsec-cli/alerts.go

@@ -0,0 +1,451 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/go-openapi/strfmt"
+	"github.com/olekukonko/tablewriter"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+var printMachine bool
+var limit *int
+
+func DecisionsFromAlert(alert *models.Alert) string {
+	ret := ""
+	var decMap = make(map[string]int)
+	for _, decision := range alert.Decisions {
+		k := *decision.Type
+		if *decision.Simulated {
+			k = fmt.Sprintf("(simul)%s", k)
+		}
+		v := decMap[k]
+		decMap[k] = v + 1
+	}
+	for k, v := range decMap {
+		if len(ret) > 0 {
+			ret += " "
+		}
+		ret += fmt.Sprintf("%s:%d", k, v)
+	}
+	return ret
+}
+
+func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
+
+	if csConfig.Cscli.Output == "raw" {
+		if printMachine {
+			fmt.Printf("id,scope,value,reason,country,as,decisions,created_at,machine\n")
+		} else {
+			fmt.Printf("id,scope,value,reason,country,as,decisions,created_at\n")
+		}
+		for _, alertItem := range *alerts {
+			if printMachine {
+				fmt.Printf("%v,%v,%v,%v,%v,%v,%v,%v,%v\n",
+					alertItem.ID,
+					*alertItem.Source.Scope,
+					*alertItem.Source.Value,
+					*alertItem.Scenario,
+					alertItem.Source.Cn,
+					alertItem.Source.AsNumber+" "+alertItem.Source.AsName,
+					DecisionsFromAlert(alertItem),
+					*alertItem.StartAt,
+					alertItem.MachineID)
+			} else {
+				fmt.Printf("%v,%v,%v,%v,%v,%v,%v,%v\n",
+					alertItem.ID,
+					*alertItem.Source.Scope,
+					*alertItem.Source.Value,
+					*alertItem.Scenario,
+					alertItem.Source.Cn,
+					alertItem.Source.AsNumber+" "+alertItem.Source.AsName,
+					DecisionsFromAlert(alertItem),
+					*alertItem.StartAt)
+			}
+
+		}
+	} else if csConfig.Cscli.Output == "json" {
+		x, _ := json.MarshalIndent(alerts, "", " ")
+		fmt.Printf("%s", string(x))
+	} else if csConfig.Cscli.Output == "human" {
+
+		table := tablewriter.NewWriter(os.Stdout)
+		if printMachine {
+			table.SetHeader([]string{"ID", "value", "reason", "country", "as", "decisions", "created_at", "machine"})
+		} else {
+			table.SetHeader([]string{"ID", "value", "reason", "country", "as", "decisions", "created_at"})
+		}
+
+		if len(*alerts) == 0 {
+			fmt.Println("No active alerts")
+			return nil
+		}
+		for _, alertItem := range *alerts {
+
+			displayVal := *alertItem.Source.Scope
+			if *alertItem.Source.Value != "" {
+				displayVal += ":" + *alertItem.Source.Value
+			}
+			if printMachine {
+				table.Append([]string{
+					strconv.Itoa(int(alertItem.ID)),
+					displayVal,
+					*alertItem.Scenario,
+					alertItem.Source.Cn,
+					alertItem.Source.AsNumber + " " + alertItem.Source.AsName,
+					DecisionsFromAlert(alertItem),
+					*alertItem.StartAt,
+					alertItem.MachineID,
+				})
+			} else {
+				table.Append([]string{
+					strconv.Itoa(int(alertItem.ID)),
+					displayVal,
+					*alertItem.Scenario,
+					alertItem.Source.Cn,
+					alertItem.Source.AsNumber + " " + alertItem.Source.AsName,
+					DecisionsFromAlert(alertItem),
+					*alertItem.StartAt,
+				})
+			}
+		}
+		table.Render() // Send output
+	}
+	return nil
+}
+
+func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
+	if csConfig.Cscli.Output == "human" {
+		fmt.Printf("\n################################################################################################\n\n")
+		scopeAndValue := *alert.Source.Scope
+		if *alert.Source.Value != "" {
+			scopeAndValue += ":" + *alert.Source.Value
+		}
+		fmt.Printf(" - ID         : %d\n", alert.ID)
+		fmt.Printf(" - Date       : %s\n", alert.CreatedAt)
+		fmt.Printf(" - Machine    : %s\n", alert.MachineID)
+		fmt.Printf(" - Simulation : %v\n", *alert.Simulated)
+		fmt.Printf(" - Reason     : %s\n", *alert.Scenario)
+		fmt.Printf(" - Events Count : %d\n", *alert.EventsCount)
+		fmt.Printf(" - Scope:Value: %s\n", scopeAndValue)
+		fmt.Printf(" - Country    : %s\n", alert.Source.Cn)
+		fmt.Printf(" - AS         : %s\n\n", alert.Source.AsName)
+		foundActive := false
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetHeader([]string{"ID", "scope:value", "action", "expiration", "created_at"})
+		for _, decision := range alert.Decisions {
+			parsedDuration, err := time.ParseDuration(*decision.Duration)
+			if err != nil {
+				log.Errorf(err.Error())
+			}
+			expire := time.Now().Add(parsedDuration)
+			if time.Now().After(expire) {
+				continue
+			}
+			foundActive = true
+			scopeAndValue := *decision.Scope
+			if *decision.Value != "" {
+				scopeAndValue += ":" + *decision.Value
+			}
+			table.Append([]string{
+				strconv.Itoa(int(decision.ID)),
+				scopeAndValue,
+				*decision.Type,
+				*decision.Duration,
+				alert.CreatedAt,
+			})
+		}
+		if foundActive {
+			fmt.Printf(" - Active Decisions  :\n")
+			table.Render() // Send output
+		}
+
+		if withDetail {
+			fmt.Printf("\n - Events  :\n")
+			for _, event := range alert.Events {
+				fmt.Printf("\n- Date: %s\n", *event.Timestamp)
+				table = tablewriter.NewWriter(os.Stdout)
+				table.SetHeader([]string{"Key", "Value"})
+				for _, meta := range event.Meta {
+					table.Append([]string{
+						meta.Key,
+						meta.Value,
+					})
+				}
+				table.Render() // Send output
+			}
+		}
+	}
+	return nil
+}
+
+func NewAlertsCmd() *cobra.Command {
+	/* ---- ALERTS COMMAND */
+	var cmdAlerts = &cobra.Command{
+		Use:   "alerts [action]",
+		Short: "Manage alerts",
+		Args:  cobra.MinimumNArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var err error
+			if csConfig.API.Client == nil {
+				log.Fatalln("There is no configuration on 'api_client:'")
+			}
+			if csConfig.API.Client.Credentials == nil {
+				log.Fatalf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath)
+			}
+			apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
+			if err != nil {
+				log.Fatalf("parsing api url: %s", apiURL)
+			}
+			Client, err = apiclient.NewClient(&apiclient.Config{
+				MachineID:     csConfig.API.Client.Credentials.Login,
+				Password:      strfmt.Password(csConfig.API.Client.Credentials.Password),
+				UserAgent:     fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
+				URL:           apiURL,
+				VersionPrefix: "v1",
+			})
+
+			if err != nil {
+				log.Fatalf("new api client: %s", err.Error())
+			}
+		},
+	}
+
+	var alertListFilter = apiclient.AlertsListOpts{
+		ScopeEquals:    new(string),
+		ValueEquals:    new(string),
+		ScenarioEquals: new(string),
+		IPEquals:       new(string),
+		RangeEquals:    new(string),
+		Since:          new(string),
+		Until:          new(string),
+		TypeEquals:     new(string),
+	}
+	limit = new(int)
+	var cmdAlertsList = &cobra.Command{
+		Use:   "list [filters]",
+		Short: "List alerts",
+		Example: `cscli alerts list
+cscli alerts list --ip 1.2.3.4
+cscli alerts list --range 1.2.3.0/24
+cscli alerts list -s crowdsecurity/ssh-bf
+cscli alerts list --type ban`,
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
+			if err := manageCliDecisionAlerts(alertListFilter.IPEquals, alertListFilter.RangeEquals,
+				alertListFilter.ScopeEquals, alertListFilter.ValueEquals); err != nil {
+				_ = cmd.Help()
+				log.Fatalf("%s", err)
+			}
+			if limit != nil {
+				alertListFilter.Limit = limit
+			}
+
+			if *alertListFilter.Until == "" {
+				alertListFilter.Until = nil
+			} else {
+				/*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/
+				if strings.HasSuffix(*alertListFilter.Until, "d") {
+					realDuration := strings.TrimSuffix(*alertListFilter.Until, "d")
+					days, err := strconv.Atoi(realDuration)
+					if err != nil {
+						cmd.Help()
+						log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *alertListFilter.Until)
+					}
+					*alertListFilter.Until = fmt.Sprintf("%d%s", days*24, "h")
+				}
+			}
+			if *alertListFilter.Since == "" {
+				alertListFilter.Since = nil
+			} else {
+				/*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/
+				if strings.HasSuffix(*alertListFilter.Since, "d") {
+					realDuration := strings.TrimSuffix(*alertListFilter.Since, "d")
+					days, err := strconv.Atoi(realDuration)
+					if err != nil {
+						cmd.Help()
+						log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *alertListFilter.Since)
+					}
+					*alertListFilter.Since = fmt.Sprintf("%d%s", days*24, "h")
+				}
+			}
+			if *alertListFilter.TypeEquals == "" {
+				alertListFilter.TypeEquals = nil
+			}
+			if *alertListFilter.ScopeEquals == "" {
+				alertListFilter.ScopeEquals = nil
+			}
+			if *alertListFilter.ValueEquals == "" {
+				alertListFilter.ValueEquals = nil
+			}
+			if *alertListFilter.ScenarioEquals == "" {
+				alertListFilter.ScenarioEquals = nil
+			}
+			if *alertListFilter.IPEquals == "" {
+				alertListFilter.IPEquals = nil
+			}
+			if *alertListFilter.RangeEquals == "" {
+				alertListFilter.RangeEquals = nil
+			}
+			alerts, _, err := Client.Alerts.List(context.Background(), alertListFilter)
+			if err != nil {
+				log.Fatalf("Unable to list alerts : %v", err.Error())
+			}
+
+			err = AlertsToTable(alerts, printMachine)
+			if err != nil {
+				log.Fatalf("unable to list alerts : %v", err.Error())
+			}
+		},
+	}
+	cmdAlertsList.Flags().SortFlags = false
+	cmdAlertsList.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
+	cmdAlertsList.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
+	cmdAlertsList.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
+	cmdAlertsList.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmdAlertsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sended alerts")
+	cmdAlertsList.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
+	cmdAlerts.AddCommand(cmdAlertsList)
+
+	var ActiveDecision *bool
+	var AlertDeleteAll bool
+	var alertDeleteFilter = apiclient.AlertsDeleteOpts{
+		ScopeEquals:    new(string),
+		ValueEquals:    new(string),
+		ScenarioEquals: new(string),
+		IPEquals:       new(string),
+		RangeEquals:    new(string),
+	}
+	var cmdAlertsDelete = &cobra.Command{
+		Use: "delete [filters] [--all]",
+		Short: `Delete alerts
+/!\ This command can be use only on the same machine than the local API.`,
+		Example: `cscli alerts delete --ip 1.2.3.4
+cscli alerts delete --range 1.2.3.0/24
+cscli alerts delete -s crowdsecurity/ssh-bf"`,
+		Args: cobra.ExactArgs(0),
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if AlertDeleteAll {
+				return
+			}
+			if *alertDeleteFilter.ScopeEquals == "" && *alertDeleteFilter.ValueEquals == "" &&
+				*alertDeleteFilter.ScenarioEquals == "" && *alertDeleteFilter.IPEquals == "" &&
+				*alertDeleteFilter.RangeEquals == "" {
+				_ = cmd.Usage()
+				log.Fatalln("At least one filter or --all must be specified")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
+			if !AlertDeleteAll {
+				if err := manageCliDecisionAlerts(alertDeleteFilter.IPEquals, alertDeleteFilter.RangeEquals,
+					alertDeleteFilter.ScopeEquals, alertDeleteFilter.ValueEquals); err != nil {
+					_ = cmd.Help()
+					log.Fatalf("%s", err)
+				}
+				if ActiveDecision != nil {
+					alertDeleteFilter.ActiveDecisionEquals = ActiveDecision
+				}
+
+				if *alertDeleteFilter.ScopeEquals == "" {
+					alertDeleteFilter.ScopeEquals = nil
+				}
+				if *alertDeleteFilter.ValueEquals == "" {
+					alertDeleteFilter.ValueEquals = nil
+				}
+				if *alertDeleteFilter.ScenarioEquals == "" {
+					alertDeleteFilter.ScenarioEquals = nil
+				}
+				if *alertDeleteFilter.IPEquals == "" {
+					alertDeleteFilter.IPEquals = nil
+				}
+				if *alertDeleteFilter.RangeEquals == "" {
+					alertDeleteFilter.RangeEquals = nil
+				}
+			} else {
+				alertDeleteFilter = apiclient.AlertsDeleteOpts{}
+			}
+			alerts, _, err := Client.Alerts.Delete(context.Background(), alertDeleteFilter)
+			if err != nil {
+				log.Fatalf("Unable to delete alerts : %v", err.Error())
+			}
+			log.Infof("%s alert(s) deleted", alerts.NbDeleted)
+
+		},
+	}
+	cmdAlertsDelete.Flags().SortFlags = false
+	cmdAlertsDelete.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmdAlertsDelete.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
+
+	cmdAlerts.AddCommand(cmdAlertsDelete)
+
+	var details bool
+	var cmdAlertsInspect = &cobra.Command{
+		Use:     "inspect <alert_id>",
+		Short:   `Show info about an alert`,
+		Example: `cscli alerts inspect 123`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) == 0 {
+				_ = cmd.Help()
+				return
+			}
+			for _, alertID := range args {
+				id, err := strconv.Atoi(alertID)
+				if err != nil {
+					log.Fatalf("bad alert id %s", alertID)
+					continue
+				}
+				alert, _, err := Client.Alerts.GetByID(context.Background(), id)
+				if err != nil {
+					log.Fatalf("can't find alert with id %s: %s", alertID, err)
+				}
+				switch csConfig.Cscli.Output {
+				case "human":
+					if err := DisplayOneAlert(alert, details); err != nil {
+						continue
+					}
+				case "json":
+					data, err := json.MarshalIndent(alert, "", "  ")
+					if err != nil {
+						log.Fatalf("unable to marshal alert with id %s: %s", alertID, err)
+					}
+					fmt.Printf("%s\n", string(data))
+				case "raw":
+					data, err := yaml.Marshal(alert)
+					if err != nil {
+						log.Fatalf("unable to marshal alert with id %s: %s", alertID, err)
+					}
+					fmt.Printf("%s\n", string(data))
+				}
+			}
+		},
+	}
+	cmdAlertsInspect.Flags().SortFlags = false
+	cmdAlertsInspect.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
+
+	cmdAlerts.AddCommand(cmdAlertsInspect)
+
+	return cmdAlerts
+}

cmd/crowdsec-cli/api.go

@@ -1,289 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/outputs"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/denisbrodbeck/machineid"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
-)
-
-var (
-	upper  = "ABCDEFGHIJKLMNOPQRSTUVWXY"
-	lower  = "abcdefghijklmnopqrstuvwxyz"
-	digits = "0123456789"
-)
-
-var (
-	userID    string // for flag parsing
-	outputCTX *outputs.Output
-)
-
-const (
-	uuid          = "/proc/sys/kernel/random/uuid"
-	apiConfigFile = "api.yaml"
-)
-
-func dumpCredentials() error {
-	if config.output == "json" {
-		credsYaml, err := json.Marshal(&outputCTX.API.Creds)
-		if err != nil {
-			log.Fatalf("Can't marshal credentials : %v", err)
-		}
-		fmt.Printf("%s\n", string(credsYaml))
-	} else {
-		credsYaml, err := yaml.Marshal(&outputCTX.API.Creds)
-		if err != nil {
-			log.Fatalf("Can't marshal credentials : %v", err)
-		}
-		fmt.Printf("%s\n", string(credsYaml))
-	}
-	return nil
-}
-
-func generatePassword(passwordLength int) string {
-	rand.Seed(time.Now().UnixNano())
-	charset := upper + lower + digits
-
-	buf := make([]byte, passwordLength)
-	buf[0] = digits[rand.Intn(len(digits))]
-	buf[1] = upper[rand.Intn(len(upper))]
-	buf[2] = lower[rand.Intn(len(lower))]
-
-	for i := 3; i < passwordLength; i++ {
-		buf[i] = charset[rand.Intn(len(charset))]
-	}
-	rand.Shuffle(len(buf), func(i, j int) {
-		buf[i], buf[j] = buf[j], buf[i]
-	})
-
-	return string(buf)
-}
-
-func pullTOP() error {
-	/*profile from cwhub*/
-	var profiles []string
-	if _, ok := cwhub.HubIdx[cwhub.SCENARIOS]; !ok || len(cwhub.HubIdx[cwhub.SCENARIOS]) == 0 {
-		log.Errorf("no loaded scenarios, can't fill profiles")
-		return fmt.Errorf("no profiles")
-	}
-	for _, item := range cwhub.HubIdx[cwhub.SCENARIOS] {
-		if item.Tainted || !item.Installed {
-			continue
-		}
-		profiles = append(profiles, item.Name)
-	}
-	outputCTX.API.Creds.Profile = strings.Join(profiles[:], ",")
-	if err := outputCTX.API.Signin(); err != nil {
-		log.Fatalf(err.Error())
-	}
-
-	ret, err := outputCTX.API.PullTop()
-	if err != nil {
-		log.Fatalf(err.Error())
-	}
-	log.Warningf("api pull returned %d entries", len(ret))
-	for _, item := range ret {
-		if _, ok := item["range_ip"]; !ok {
-			continue
-		}
-		if _, ok := item["scenario"]; !ok {
-			continue
-		}
-
-		if _, ok := item["action"]; !ok {
-			continue
-		}
-		if _, ok := item["expiration"]; !ok {
-			continue
-		}
-		if _, ok := item["country"]; !ok {
-			item["country"] = ""
-		}
-		if _, ok := item["as_org"]; !ok {
-			item["as_org"] = ""
-		}
-		if _, ok := item["as_num"]; !ok {
-			item["as_num"] = ""
-		}
-		var signalOcc types.SignalOccurence
-		signalOcc, err = simpleBanToSignal(item["range_ip"], item["scenario"], item["expiration"], item["action"], item["as_name"], item["as_num"], item["country"], "api")
-		if err != nil {
-			return fmt.Errorf("failed to convert ban to signal : %s", err)
-		}
-		if err := outputCTX.Insert(signalOcc); err != nil {
-			log.Fatalf("Unable to write pull to Database : %+s", err.Error())
-		}
-	}
-	outputCTX.Flush()
-	log.Infof("Wrote %d bans from api to database.", len(ret))
-	return nil
-}
-
-func NewAPICmd() *cobra.Command {
-
-	var cmdAPI = &cobra.Command{
-		Use:   "api [action]",
-		Short: "Crowdsec API interaction",
-		Long: `
-Allow to register your machine into crowdsec API to send and receive signal.
-		`,
-		Example: `
-cscli api register      # Register to Crowdsec API
-cscli api pull          # Pull malevolant IPs from Crowdsec API
-cscli api reset         # Reset your machines credentials
-cscli api enroll        # Enroll your machine to the user account you created on Crowdsec backend
-cscli api credentials   # Display your API credentials
-`,
-		Args: cobra.MinimumNArgs(1),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			outputConfig := outputs.OutputFactory{
-				BackendFolder: config.BackendPluginFolder,
-				Flush:         false,
-			}
-			outputCTX, err = outputs.NewOutput(&outputConfig)
-			if err != nil {
-				return err
-			}
-
-			err = outputCTX.LoadAPIConfig(path.Join(config.InstallFolder, apiConfigFile))
-			if err != nil {
-				return err
-			}
-			return nil
-		},
-	}
-
-	var cmdAPIRegister = &cobra.Command{
-		Use:   "register",
-		Short: "Register on Crowdsec API",
-		Long: `This command will register your machine to crowdsec API to allow you to receive list of malveolent IPs. 
-		The printed machine_id and password should be added to your api.yaml file.`,
-		Example: `cscli api register`,
-		Args:    cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			id, err := machineid.ID()
-			if err != nil {
-				log.Debugf("failed to get machine-id with usual files : %s", err)
-			}
-			if id == "" || err != nil {
-				bID, err := ioutil.ReadFile(uuid)
-				if err != nil {
-					log.Fatalf("can'get a valid machine_id")
-				}
-				id = string(bID)
-				id = strings.ReplaceAll(id, "-", "")[:32]
-			}
-			password := generatePassword(64)
-
-			if err := outputCTX.API.RegisterMachine(fmt.Sprintf("%s%s", id, generatePassword(16)), password); err != nil {
-				log.Fatalf(err.Error())
-			}
-			fmt.Printf("machine_id: %s\n", outputCTX.API.Creds.User)
-			fmt.Printf("password: %s\n", outputCTX.API.Creds.Password)
-			fmt.Printf("#You need to append these credentials in /etc/crowdsec/config/api.yaml")
-		},
-	}
-
-	var cmdAPIEnroll = &cobra.Command{
-		Use:     "enroll",
-		Short:   "Associate your machine to an existing crowdsec user",
-		Long:    `Enrolling your machine into your user account will allow for more accurate lists and threat detection. See website to create user account.`,
-		Example: `cscli api enroll -u 1234567890ffff`,
-		Args:    cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := outputCTX.API.Signin(); err != nil {
-				log.Fatalf("unable to signin : %s", err)
-			}
-			if err := outputCTX.API.Enroll(userID); err != nil {
-				log.Fatalf(err.Error())
-			}
-		},
-	}
-
-	var cmdAPIResetPassword = &cobra.Command{
-		Use:     "reset",
-		Short:   "Reset password on CrowdSec API",
-		Long:    `Attempts to reset your credentials to the API.`,
-		Example: `cscli api reset`,
-		Args:    cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			id, err := machineid.ID()
-			if err != nil {
-				log.Debugf("failed to get machine-id with usual files : %s", err)
-			}
-			if id == "" || err != nil {
-				bID, err := ioutil.ReadFile(uuid)
-				if err != nil {
-					log.Fatalf("can'get a valid machine_id")
-				}
-				id = string(bID)
-				id = strings.ReplaceAll(id, "-", "")[:32]
-			}
-
-			password := generatePassword(64)
-			if err := outputCTX.API.ResetPassword(fmt.Sprintf("%s%s", id, generatePassword(16)), password); err != nil {
-				log.Fatalf(err.Error())
-			}
-			fmt.Printf("machine_id: %s\n", outputCTX.API.Creds.User)
-			fmt.Printf("password: %s\n", outputCTX.API.Creds.Password)
-		},
-	}
-
-	var cmdAPIPull = &cobra.Command{
-		Use:     "pull",
-		Short:   "Pull crowdsec API TopX",
-		Long:    `Pulls a list of malveolent IPs relevant to your situation and add them into the local ban database.`,
-		Example: `cscli api pull`,
-		Args:    cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf(err.Error())
-			}
-			err := pullTOP()
-			if err != nil {
-				log.Fatalf(err.Error())
-			}
-		},
-	}
-
-	var cmdAPICreds = &cobra.Command{
-		Use:     "credentials",
-		Short:   "Display api credentials",
-		Long:    ``,
-		Example: `cscli api credentials`,
-		Args:    cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := dumpCredentials(); err != nil {
-				log.Fatalf(err.Error())
-			}
-		},
-	}
-
-	cmdAPI.AddCommand(cmdAPICreds)
-	cmdAPIEnroll.Flags().StringVarP(&userID, "user", "u", "", "User ID (required)")
-	if err := cmdAPIEnroll.MarkFlagRequired("user"); err != nil {
-		log.Errorf("'user' flag : %s", err)
-	}
-	cmdAPI.AddCommand(cmdAPIEnroll)
-	cmdAPI.AddCommand(cmdAPIResetPassword)
-	cmdAPI.AddCommand(cmdAPIRegister)
-	cmdAPI.AddCommand(cmdAPIPull)
-	return cmdAPI
-}

cmd/crowdsec-cli/backup-restore.go

@@ -1,556 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwapi"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/outputs"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-//it's a rip of the cli version, but in silent-mode
-func silenceInstallItem(name string, obtype string) (string, error) {
-	for _, it := range cwhub.HubIdx[obtype] {
-		if it.Name == name {
-			if download_only && it.Downloaded && it.UpToDate {
-				return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
-			}
-			it, err := cwhub.DownloadLatest(it, cwhub.Hubdir, force_install, config.DataFolder)
-			if err != nil {
-				return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
-			}
-			cwhub.HubIdx[obtype][it.Name] = it
-			if download_only {
-				return fmt.Sprintf("Downloaded %s to %s", it.Name, cwhub.Hubdir+"/"+it.RemotePath), nil
-			}
-			it, err = cwhub.EnableItem(it, cwhub.Installdir, cwhub.Hubdir)
-			if err != nil {
-				return "", fmt.Errorf("error while enabled %s : %v", it.Name, err)
-			}
-			cwhub.HubIdx[obtype][it.Name] = it
-
-			return fmt.Sprintf("Enabled %s", it.Name), nil
-		}
-	}
-	return "", fmt.Errorf("%s not found in hub index", name)
-}
-
-/*help to copy the file, ioutil doesn't offer the feature*/
-
-func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-	defer func() {
-		cerr := out.Close()
-		if err == nil {
-			err = cerr
-		}
-	}()
-	if _, err = io.Copy(out, in); err != nil {
-		return
-	}
-	err = out.Sync()
-	return
-}
-
-/*copy the file, ioutile doesn't offer the feature*/
-func copyFile(sourceSymLink, destinationFile string) (err error) {
-
-	sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
-	if err != nil {
-		log.Infof("Not a symlink : %s", err)
-		sourceFile = sourceSymLink
-	}
-
-	sourceFileStat, err := os.Stat(sourceFile)
-	if err != nil {
-		return
-	}
-	if !sourceFileStat.Mode().IsRegular() {
-		// cannot copy non-regular files (e.g., directories,
-		// symlinks, devices, etc.)
-		return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
-	}
-	destinationFileStat, err := os.Stat(destinationFile)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return
-		}
-	} else {
-		if !(destinationFileStat.Mode().IsRegular()) {
-			return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
-		}
-		if os.SameFile(sourceFileStat, destinationFileStat) {
-			return
-		}
-	}
-	if err = os.Link(sourceFile, destinationFile); err == nil {
-		return
-	}
-	err = copyFileContents(sourceFile, destinationFile)
-	return
-}
-
-/*given a backup directory, restore configs (parser,collections..) both tainted and untainted.
-as well attempts to restore api credentials after verifying the existing ones aren't good
-finally restores the acquis.yaml file*/
-func restoreFromDirectory(source string) error {
-	var err error
-
-	/*restore simulation configuration*/
-	backSimul := fmt.Sprintf("%s/simulation.yaml", source)
-	if _, err = os.Stat(backSimul); err == nil {
-		if err = copyFile(backSimul, config.SimulationCfgPath); err != nil {
-			return fmt.Errorf("failed copy %s to %s : %s", backSimul, config.SimulationCfgPath, err)
-		}
-	}
-
-	/*restore scenarios etc.*/
-	for _, itype := range cwhub.ItemTypes {
-		itemDirectory := fmt.Sprintf("%s/%s/", source, itype)
-		if _, err = os.Stat(itemDirectory); err != nil {
-			log.Infof("no %s in backup", itype)
-			continue
-		}
-		/*restore the upstream items*/
-		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
-		file, err := ioutil.ReadFile(upstreamListFN)
-		if err != nil {
-			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
-		}
-		var upstreamList []string
-		err = json.Unmarshal([]byte(file), &upstreamList)
-		if err != nil {
-			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
-		}
-		for _, toinstall := range upstreamList {
-			label, err := silenceInstallItem(toinstall, itype)
-			if err != nil {
-				log.Errorf("Error while installing %s : %s", toinstall, err)
-			} else if label != "" {
-				log.Infof("Installed %s : %s", toinstall, label)
-			} else {
-				log.Printf("Installed %s : ok", toinstall)
-			}
-		}
-		/*restore the local and tainted items*/
-		files, err := ioutil.ReadDir(itemDirectory)
-		if err != nil {
-			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
-		}
-		for _, file := range files {
-			//dir are stages, keep track
-			if !file.IsDir() {
-				continue
-			}
-			stage := file.Name()
-			stagedir := fmt.Sprintf("%s/%s/%s/", config.InstallFolder, itype, stage)
-			log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
-			if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
-				return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
-			}
-			/*find items*/
-			ifiles, err := ioutil.ReadDir(itemDirectory + "/" + stage + "/")
-			if err != nil {
-				return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
-			}
-			//finaly copy item
-			for _, tfile := range ifiles {
-				log.Infof("Going to restore local/tainted [%s]", tfile.Name())
-				sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
-				destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
-				if err = copyFile(sourceFile, destinationFile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
-				}
-				log.Infof("restored %s to %s", sourceFile, destinationFile)
-			}
-		}
-	}
-	/*restore api credentials*/
-	//check if credentials exists :
-	// - if no, restore
-	// - if yes, try them :
-	//		- if it works, left untouched
-	//		- if not, restore
-	// -> try login
-	if err := restoreAPICreds(source); err != nil {
-		return fmt.Errorf("failed to restore api credentials : %s", err)
-	}
-	/*
-		Restore acquis
-	*/
-	yamlAcquisFile := fmt.Sprintf("%s/acquis.yaml", config.InstallFolder)
-	bac := fmt.Sprintf("%s/acquis.yaml", source)
-	if err = copyFile(bac, yamlAcquisFile); err != nil {
-		return fmt.Errorf("failed copy %s to %s : %s", bac, yamlAcquisFile, err)
-	}
-	log.Infof("Restore acquis to %s", yamlAcquisFile)
-
-	/* Restore plugins configuration */
-	var pluginsConfigFile []string
-	walkErr := filepath.Walk(fmt.Sprintf("%s/plugins/backend/", source), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return fmt.Errorf("walk error : %s", err)
-		}
-		fi, err := os.Stat(path)
-		if err != nil {
-			return fmt.Errorf("unable to stats file '%s' : %s", path, err)
-		}
-		mode := fi.Mode()
-		if mode.IsRegular() {
-			pluginsConfigFile = append(pluginsConfigFile, path)
-		}
-		return nil
-	})
-	if walkErr != nil {
-		return fmt.Errorf("error while listing folder '%s' : %s", fmt.Sprintf("%s/plugins/backend/", source), walkErr)
-	}
-
-	if err := os.MkdirAll(outputCTX.Config.BackendFolder, os.ModePerm); err != nil {
-		return fmt.Errorf("error while creating backup folder dir %s : %s", outputCTX.Config.BackendFolder, err)
-	}
-
-	for _, file := range pluginsConfigFile {
-		_, filename := path.Split(file)
-		backupFile := fmt.Sprintf("%s/%s", outputCTX.Config.BackendFolder, filename)
-		log.Printf("Restoring  '%s' to '%s'", file, backupFile)
-		if err := copyFile(file, backupFile); err != nil {
-			return fmt.Errorf("error while copying '%s' to '%s' : %s", file, backupFile, err)
-		}
-	}
-
-	return nil
-}
-
-func restoreAPICreds(source string) error {
-	var err error
-
-	/*check existing configuration*/
-	apiyaml := path.Join(config.InstallFolder, apiConfigFile)
-
-	api := &cwapi.ApiCtx{}
-	if err = api.LoadConfig(apiyaml); err != nil {
-		return fmt.Errorf("unable to load api config %s : %s", apiyaml, err)
-	}
-	if api.Creds.User != "" {
-		log.Infof("Credentials present in existing configuration, try before override")
-		err := api.Signin()
-		if err == nil {
-			log.Infof("Credentials present allow authentication, don't override !")
-			return nil
-		} else {
-			log.Infof("Credentials aren't valid : %s", err)
-		}
-	}
-	/*existing config isn't good, override it !*/
-	ret, err := ioutil.ReadFile(path.Join(source, "api_creds.json"))
-	if err != nil {
-		return fmt.Errorf("failed to read api creds from save : %s", err)
-	}
-	if err := json.Unmarshal(ret, &api.Creds); err != nil {
-		return fmt.Errorf("failed unmarshaling saved credentials : %s", err)
-	}
-	api.CfgUser = api.Creds.User
-	api.CfgPassword = api.Creds.Password
-	/*override the existing yaml file*/
-	if err := api.WriteConfig(apiyaml); err != nil {
-		return fmt.Errorf("failed writing to %s : %s", apiyaml, err)
-	} else {
-		log.Infof("Overwritting %s with backup info", apiyaml)
-	}
-
-	/*reload to check everything is safe*/
-	if err = api.LoadConfig(apiyaml); err != nil {
-		return fmt.Errorf("unable to load api config %s : %s", apiyaml, err)
-	}
-
-	if err := api.Signin(); err != nil {
-		log.Errorf("Failed to authenticate after credentials restaurtion : %v", err)
-	} else {
-		log.Infof("Successfully auth to API after credentials restauration")
-	}
-
-	return nil
-}
-
-func backupToDirectory(target string) error {
-	var itemDirectory string
-	var upstreamParsers []string
-	var err error
-	if target == "" {
-		return fmt.Errorf("target directory can't be empty")
-	}
-	log.Warningf("Starting configuration backup")
-	_, err = os.Stat(target)
-	if err == nil {
-		return fmt.Errorf("%s already exists", target)
-	}
-	if err = os.MkdirAll(target, os.ModePerm); err != nil {
-		return fmt.Errorf("error while creating %s : %s", target, err)
-	}
-	/*
-		backup configurations :
-			- parers, scenarios, collections, postoverflows
-			- simulation configuration
-	*/
-	if config.SimulationCfgPath != "" {
-		backSimul := fmt.Sprintf("%s/simulation.yaml", target)
-		if err = copyFile(config.SimulationCfgPath, backSimul); err != nil {
-			return fmt.Errorf("failed copy %s to %s : %s", config.SimulationCfgPath, backSimul, err)
-		}
-	}
-
-	for _, itemType := range cwhub.ItemTypes {
-		clog := log.WithFields(log.Fields{
-			"type": itemType,
-		})
-		if _, ok := cwhub.HubIdx[itemType]; ok {
-			itemDirectory = fmt.Sprintf("%s/%s/", target, itemType)
-			if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
-				return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
-			}
-			upstreamParsers = []string{}
-			stage := ""
-			for k, v := range cwhub.HubIdx[itemType] {
-				clog = clog.WithFields(log.Fields{
-					"file": v.Name,
-				})
-				if !v.Installed { //only backup installed ones
-					clog.Debugf("[%s] : not installed", k)
-					continue
-				}
-
-				//for the local/tainted ones, we backup the full file
-				if v.Tainted || v.Local || !v.UpToDate {
-					//we need to backup stages for parsers
-					if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
-						tmp := strings.Split(v.LocalPath, "/")
-						stage = "/" + tmp[len(tmp)-2] + "/"
-						fstagedir := fmt.Sprintf("%s%s", itemDirectory, stage)
-						if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
-							return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
-						}
-					}
-					clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
-					tfile := fmt.Sprintf("%s%s%s", itemDirectory, stage, v.FileName)
-					//clog.Infof("item : %s", spew.Sdump(v))
-					if err = copyFile(v.LocalPath, tfile); err != nil {
-						return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
-					}
-					clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
-					continue
-				}
-				clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
-				clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
-				upstreamParsers = append(upstreamParsers, v.Name)
-			}
-			//write the upstream items
-			upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
-			upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
-			if err != nil {
-				return fmt.Errorf("failed marshaling upstream parsers : %s", err)
-			}
-			err = ioutil.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
-			if err != nil {
-				return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
-			}
-			clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
-
-		} else {
-			clog.Infof("No %s to backup.", itemType)
-		}
-	}
-	/*
-		Backup acquis
-	*/
-	yamlAcquisFile := fmt.Sprintf("%s/acquis.yaml", config.InstallFolder)
-	bac := fmt.Sprintf("%s/acquis.yaml", target)
-	if err = copyFile(yamlAcquisFile, bac); err != nil {
-		return fmt.Errorf("failed copy %s to %s : %s", yamlAcquisFile, bac, err)
-	}
-	log.Infof("Saved acquis to %s", bac)
-	/*
-		Backup default.yaml
-	*/
-	defyaml := fmt.Sprintf("%s/default.yaml", config.InstallFolder)
-	bac = fmt.Sprintf("%s/default.yaml", target)
-	if err = copyFile(defyaml, bac); err != nil {
-		return fmt.Errorf("failed copy %s to %s : %s", yamlAcquisFile, bac, err)
-	}
-	log.Infof("Saved default yaml to %s", bac)
-	/*
-		Backup API info
-	*/
-	if outputCTX == nil {
-		log.Fatalf("no API output context, won't save api credentials")
-	}
-	outputCTX.API = &cwapi.ApiCtx{}
-	if err = outputCTX.API.LoadConfig(path.Join(config.InstallFolder, apiConfigFile)); err != nil {
-		return fmt.Errorf("unable to load api config %s : %s", path.Join(config.InstallFolder, apiConfigFile), err)
-	}
-	credsYaml, err := json.Marshal(&outputCTX.API.Creds)
-	if err != nil {
-		log.Fatalf("can't marshal credentials : %v", err)
-	}
-	apiCredsDumped := fmt.Sprintf("%s/api_creds.json", target)
-	err = ioutil.WriteFile(apiCredsDumped, credsYaml, 0600)
-	if err != nil {
-		return fmt.Errorf("unable to write credentials to %s : %s", apiCredsDumped, err)
-	}
-	log.Infof("Saved configuration to %s", target)
-
-	/* Backup plugins configuration */
-	var pluginsConfigFile []string
-	walkErr := filepath.Walk(outputCTX.Config.BackendFolder, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return fmt.Errorf("walk error : %s", err)
-		}
-		fi, err := os.Stat(path)
-		if err != nil {
-			return fmt.Errorf("unable to stats file '%s' : %s", path, err)
-		}
-		mode := fi.Mode()
-		if mode.IsRegular() {
-			pluginsConfigFile = append(pluginsConfigFile, path)
-		}
-		return nil
-	})
-	if walkErr != nil {
-		return fmt.Errorf("error while listing folder '%s' : %s", outputCTX.Config.BackendFolder, walkErr)
-	}
-
-	targetDir := fmt.Sprintf("%s/plugins/backend/", target)
-	if err := os.MkdirAll(targetDir, os.ModePerm); err != nil {
-		return fmt.Errorf("error while creating backup folder dir %s : %s", targetDir, err)
-	}
-
-	for _, file := range pluginsConfigFile {
-		_, filename := path.Split(file)
-		backupFile := fmt.Sprintf("%s/plugins/backend/%s", target, filename)
-		if err := copyFile(file, backupFile); err != nil {
-			return fmt.Errorf("unable to copy file '%s' to '%s' : %s", file, backupFile, err)
-		}
-	}
-
-	return nil
-}
-
-func NewBackupCmd() *cobra.Command {
-	var cmdBackup = &cobra.Command{
-		Use:   "backup [save|restore] <directory>",
-		Short: "Backup or restore configuration (api, parsers, scenarios etc.) to/from directory",
-		Long:  `This command is here to help you save and/or restore crowdsec configurations to simple replication`,
-		Example: `cscli backup save ./my-backup
-cscli backup restore ./my-backup`,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-			return nil
-		},
-	}
-
-	var cmdBackupSave = &cobra.Command{
-		Use:   "save <directory>",
-		Short: "Backup configuration (api, parsers, scenarios etc.) to directory",
-		Long: `backup command will try to save all relevant informations to crowdsec config, including :
-
-- List of scenarios, parsers, postoverflows and collections that are up-to-date
-
-- Actual backup of tainted/local/out-of-date scenarios, parsers, postoverflows and collections
-
-- Backup of API credentials
-
-- Backup of acquisition configuration
-		
-		`,
-		Example: `cscli backup save ./my-backup`,
-		Args:    cobra.ExactArgs(1),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-			return nil
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			var err error
-
-			outputConfig := outputs.OutputFactory{
-				BackendFolder: config.BackendPluginFolder,
-				Flush:         false,
-			}
-			outputCTX, err = outputs.NewOutput(&outputConfig)
-			if err != nil {
-				log.Fatalf("Failed to load output plugins : %v", err)
-			}
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if err := backupToDirectory(args[0]); err != nil {
-				log.Fatalf("Failed backuping to %s : %s", args[0], err)
-			}
-		},
-	}
-	cmdBackup.AddCommand(cmdBackupSave)
-
-	var cmdBackupRestore = &cobra.Command{
-		Use:   "restore <directory>",
-		Short: "Restore configuration (api, parsers, scenarios etc.) from directory",
-		Long: `restore command will try to restore all saved information from <directory> to yor local setup, including :
-
-- Installation of up-to-date scenarios/parsers/... via cscli
-
-- Restauration of tainted/local/out-of-date scenarios/parsers/... file
-
-- Restauration of API credentials (if the existing ones aren't working)
-
-- Restauration of acqusition configuration
-`,
-		Example: `cscli backup restore ./my-backup`,
-		Args:    cobra.ExactArgs(1),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-			return nil
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			var err error
-
-			outputConfig := outputs.OutputFactory{
-				BackendFolder: config.BackendPluginFolder,
-				Flush:         false,
-			}
-			outputCTX, err = outputs.NewOutput(&outputConfig)
-			if err != nil {
-				log.Fatalf("Failed to load output plugins : %v", err)
-			}
-
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			if err := restoreFromDirectory(args[0]); err != nil {
-				log.Fatalf("failed restoring from %s : %s", args[0], err)
-			}
-		},
-	}
-	cmdBackup.AddCommand(cmdBackupRestore)
-
-	return cmdBackup
-}

cmd/crowdsec-cli/ban.go

@@ -1,470 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"net"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/crowdsecurity/crowdsec/pkg/outputs"
-	"github.com/crowdsecurity/crowdsec/pkg/parser"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/olekukonko/tablewriter"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-var remediationType string
-var atTime string
-
-//user supplied filters
-var ipFilter, rangeFilter, reasonFilter, countryFilter, asFilter string
-var displayLimit int
-var displayAPI, displayALL bool
-
-func simpleBanToSignal(targetIP string, reason string, expirationStr string, action string, asName string, asNum string, country string, banSource string) (types.SignalOccurence, error) {
-	var signalOcc types.SignalOccurence
-
-	expiration, err := time.ParseDuration(expirationStr)
-	if err != nil {
-		return signalOcc, err
-	}
-
-	asOrgInt := 0
-	if asNum != "" {
-		asOrgInt, err = strconv.Atoi(asNum)
-		if err != nil {
-			log.Infof("Invalid as value %s : %s", asNum, err)
-		}
-	}
-
-	banApp := types.BanApplication{
-		MeasureSource: banSource,
-		MeasureType:   action,
-		Until:         time.Now().Add(expiration),
-		IpText:        targetIP,
-		TargetCN:      country,
-		TargetAS:      asOrgInt,
-		TargetASName:  asName,
-		Reason:        reason,
-	}
-	var parsedIP net.IP
-	var parsedRange *net.IPNet
-	if strings.Contains(targetIP, "/") {
-		if _, parsedRange, err = net.ParseCIDR(targetIP); err != nil {
-			return signalOcc, fmt.Errorf("'%s' is not a valid CIDR", targetIP)
-		}
-		if parsedRange == nil {
-			return signalOcc, fmt.Errorf("unable to parse network : %s", err)
-		}
-		banApp.StartIp = types.IP2Int(parsedRange.IP)
-		banApp.EndIp = types.IP2Int(types.LastAddress(parsedRange))
-	} else {
-		parsedIP = net.ParseIP(targetIP)
-		if parsedIP == nil {
-			return signalOcc, fmt.Errorf("'%s' is not a valid IP", targetIP)
-		}
-		banApp.StartIp = types.IP2Int(parsedIP)
-		banApp.EndIp = types.IP2Int(parsedIP)
-	}
-
-	var banApps = make([]types.BanApplication, 0)
-	banApps = append(banApps, banApp)
-	signalOcc = types.SignalOccurence{
-		Scenario:                            reason,
-		Events_count:                        1,
-		Start_at:                            time.Now(),
-		Stop_at:                             time.Now(),
-		BanApplications:                     banApps,
-		Source_ip:                           targetIP,
-		Source_AutonomousSystemNumber:       asNum,
-		Source_AutonomousSystemOrganization: asName,
-		Source_Country:                      country,
-	}
-	return signalOcc, nil
-}
-
-func filterBans(bans []map[string]string) ([]map[string]string, error) {
-
-	var retBans []map[string]string
-
-	for _, ban := range bans {
-		var banIP net.IP
-		var banRange *net.IPNet
-		var keep bool = true
-		var err error
-
-		if ban["iptext"] != "" {
-			if strings.Contains(ban["iptext"], "/") {
-				log.Debugf("%s is a range", ban["iptext"])
-				banIP, banRange, err = net.ParseCIDR(ban["iptext"])
-				if err != nil {
-					log.Warningf("failed to parse range '%s' from database : %s", ban["iptext"], err)
-				}
-			} else {
-				log.Debugf("%s is IP", ban["iptext"])
-				banIP = net.ParseIP(ban["iptext"])
-			}
-		}
-
-		if ipFilter != "" {
-			var filterBinIP net.IP = net.ParseIP(ipFilter)
-
-			if banRange != nil {
-				if banRange.Contains(filterBinIP) {
-					log.Debugf("[keep] ip filter is set, and range contains ip")
-					keep = true
-				} else {
-					log.Debugf("[discard] ip filter is set, and range doesn't contain ip")
-					keep = false
-				}
-			} else {
-				if ipFilter == ban["iptext"] {
-					log.Debugf("[keep] (ip) %s == %s", ipFilter, ban["iptext"])
-					keep = true
-				} else {
-					log.Debugf("[discard] (ip) %s == %s", ipFilter, ban["iptext"])
-					keep = false
-				}
-			}
-		}
-		if rangeFilter != "" {
-			_, filterBinRange, err := net.ParseCIDR(rangeFilter)
-			if err != nil {
-				return nil, fmt.Errorf("failed to parse range '%s' : %s", rangeFilter, err)
-			}
-			if filterBinRange.Contains(banIP) {
-				log.Debugf("[keep] range filter %s contains %s", rangeFilter, banIP.String())
-				keep = true
-			} else {
-				log.Debugf("[discard] range filter %s doesn't contain %s", rangeFilter, banIP.String())
-				keep = false
-			}
-		}
-		if reasonFilter != "" {
-			if strings.Contains(ban["reason"], reasonFilter) {
-				log.Debugf("[keep] reason filter %s matches %s", reasonFilter, ban["reason"])
-				keep = true
-			} else {
-				log.Debugf("[discard] reason filter %s doesn't match %s", reasonFilter, ban["reason"])
-				keep = false
-			}
-		}
-
-		if countryFilter != "" {
-			if ban["cn"] == countryFilter {
-				log.Debugf("[keep] country filter %s matches %s", countryFilter, ban["cn"])
-				keep = true
-			} else {
-				log.Debugf("[discard] country filter %s matches %s", countryFilter, ban["cn"])
-				keep = false
-			}
-		}
-
-		if asFilter != "" {
-			if strings.Contains(ban["as"], asFilter) {
-				log.Debugf("[keep] AS filter %s matches %s", asFilter, ban["as"])
-				keep = true
-			} else {
-				log.Debugf("[discard] AS filter %s doesn't match %s", asFilter, ban["as"])
-				keep = false
-			}
-		}
-
-		if keep {
-			retBans = append(retBans, ban)
-		} else {
-			log.Debugf("[discard] discard %v", ban)
-		}
-	}
-	return retBans, nil
-}
-
-func BanList() error {
-	at := time.Now()
-	if atTime != "" {
-		_, at = parser.GenDateParse(atTime)
-		if at.IsZero() {
-			return fmt.Errorf("unable to parse date '%s'", atTime)
-		}
-	}
-	ret, err := outputCTX.ReadAT(at)
-	if err != nil {
-		return fmt.Errorf("unable to get records from Database : %v", err)
-	}
-	ret, err = filterBans(ret)
-	if err != nil {
-		log.Errorf("Error while filtering : %s", err)
-	}
-	if config.output == "raw" {
-		fmt.Printf("source,ip,reason,bans,action,country,as,events_count,expiration\n")
-		for _, rm := range ret {
-			fmt.Printf("%s,%s,%s,%s,%s,%s,%s,%s,%s\n", rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"])
-		}
-	} else if config.output == "json" {
-		x, _ := json.MarshalIndent(ret, "", " ")
-		fmt.Printf("%s", string(x))
-	} else if config.output == "human" {
-
-		uniqAS := map[string]bool{}
-		uniqCN := map[string]bool{}
-
-		table := tablewriter.NewWriter(os.Stdout)
-		table.SetHeader([]string{"Source", "Ip", "Reason", "Bans", "Action", "Country", "AS", "Events", "Expiration"})
-
-		dispcount := 0
-		apicount := 0
-		for _, rm := range ret {
-			if !displayAPI && rm["source"] == "api" {
-				apicount++
-				if _, ok := uniqAS[rm["as"]]; !ok {
-					uniqAS[rm["as"]] = true
-				}
-				if _, ok := uniqCN[rm["cn"]]; !ok {
-					uniqCN[rm["cn"]] = true
-				}
-			}
-			if displayALL {
-				if rm["source"] == "api" {
-					if displayAPI {
-						table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
-						dispcount++
-						continue
-					}
-				} else {
-					table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
-					dispcount++
-					continue
-				}
-			} else if dispcount < displayLimit {
-				if displayAPI {
-					if rm["source"] == "api" {
-						table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
-						dispcount++
-						continue
-					}
-				} else {
-					if rm["source"] != "api" {
-						table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
-						dispcount++
-						continue
-					}
-				}
-			}
-		}
-		if dispcount > 0 {
-			if !displayAPI {
-				fmt.Printf("%d local decisions:\n", dispcount)
-			} else if displayAPI && !displayALL {
-				fmt.Printf("%d decision from API\n", dispcount)
-			} else if displayALL && displayAPI {
-				fmt.Printf("%d decision from crowdsec and API\n", dispcount)
-			}
-			table.Render() // Send output
-			if dispcount > displayLimit && !displayALL {
-				fmt.Printf("Additional records stripped.\n")
-			}
-		} else {
-			if displayAPI {
-				fmt.Println("No API decisions")
-			} else {
-				fmt.Println("No local decisions")
-			}
-		}
-		if !displayAPI {
-			fmt.Printf("And %d records from API, %d distinct AS, %d distinct countries\n", apicount, len(uniqAS), len(uniqCN))
-		}
-	}
-	return nil
-}
-
-func BanAdd(target string, duration string, reason string, action string) error {
-	var signalOcc types.SignalOccurence
-	var err error
-
-	signalOcc, err = simpleBanToSignal(target, reason, duration, action, "", "", "", "cli")
-	if err != nil {
-		return fmt.Errorf("unable to insert ban : %v", err)
-	}
-	err = outputCTX.Insert(signalOcc)
-	if err != nil {
-		return err
-	}
-	err = outputCTX.Flush()
-	if err != nil {
-		return err
-	}
-	log.Infof("%s %s for %s (%s)", action, target, duration, reason)
-	return nil
-}
-
-func NewBanCmds() *cobra.Command {
-	/*TODO : add a remediation type*/
-	var cmdBan = &cobra.Command{
-		Use:   "ban [command] <target> <duration> <reason>",
-		Short: "Manage bans/mitigations",
-		Long: `This is the main interaction point with local ban database for humans.
-
-You can add/delete/list or flush current bans in your local ban DB.`,
-		Args: cobra.MinimumNArgs(1),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before using bans")
-			}
-
-			outputConfig := outputs.OutputFactory{
-				BackendFolder: config.BackendPluginFolder,
-				Flush:         false,
-			}
-
-			outputCTX, err = outputs.NewOutput(&outputConfig)
-			if err != nil {
-				return fmt.Errorf(err.Error())
-			}
-			return nil
-		},
-	}
-	cmdBan.PersistentFlags().StringVar(&remediationType, "remediation", "ban", "Set specific remediation type : ban|slow|captcha")
-	cmdBan.Flags().SortFlags = false
-	cmdBan.PersistentFlags().SortFlags = false
-
-	var cmdBanAdd = &cobra.Command{
-		Use:   "add [ip|range] <target> <duration> <reason>",
-		Short: "Adds a ban against a given ip/range for the provided duration",
-		Long: `
-Allows to add a ban against a specific ip or range target for a specific duration.  
-
-The duration argument can be expressed in seconds(s), minutes(m) or hours (h).
-		
-See [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) for more informations.`,
-		Example: `cscli ban add ip 1.2.3.4 24h "scan"  
-cscli ban add range 1.2.3.0/24 24h "the whole range"`,
-		Args: cobra.MinimumNArgs(4),
-	}
-	cmdBan.AddCommand(cmdBanAdd)
-	var cmdBanAddIp = &cobra.Command{
-		Use:     "ip <target> <duration> <reason>",
-		Short:   "Adds the specific ip to the ban db",
-		Long:    `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration), expressed in s/m/h.`,
-		Example: `cscli ban add ip 1.2.3.4 12h "the scan"`,
-		Args:    cobra.MinimumNArgs(3),
-		Run: func(cmd *cobra.Command, args []string) {
-			reason := strings.Join(args[2:], " ")
-			if err := BanAdd(args[0], args[1], reason, remediationType); err != nil {
-				log.Fatalf("failed to add ban to database : %v", err)
-			}
-		},
-	}
-	cmdBanAdd.AddCommand(cmdBanAddIp)
-	var cmdBanAddRange = &cobra.Command{
-		Use:     "range <target> <duration> <reason>",
-		Short:   "Adds the specific ip to the ban db",
-		Long:    `Duration must be [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) compatible, expressed in s/m/h.`,
-		Example: `cscli ban add range 1.2.3.0/24 12h "the whole range"`,
-		Args:    cobra.MinimumNArgs(3),
-		Run: func(cmd *cobra.Command, args []string) {
-			reason := strings.Join(args[2:], " ")
-			if err := BanAdd(args[0], args[1], reason, remediationType); err != nil {
-				log.Fatalf("failed to add ban to database : %v", err)
-			}
-		},
-	}
-	cmdBanAdd.AddCommand(cmdBanAddRange)
-	var cmdBanDel = &cobra.Command{
-		Use:   "del [command] <target>",
-		Short: "Delete bans from db",
-		Long:  "The removal of the bans can be applied on a single IP address or directly on a IP range.",
-		Example: `cscli ban del ip 1.2.3.4  
-cscli ban del range 1.2.3.0/24`,
-		Args: cobra.MinimumNArgs(2),
-	}
-	cmdBan.AddCommand(cmdBanDel)
-
-	var cmdBanFlush = &cobra.Command{
-		Use:     "flush",
-		Short:   "Fush ban DB",
-		Example: `cscli ban flush`,
-		Args:    cobra.NoArgs,
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := outputCTX.DeleteAll(); err != nil {
-				log.Fatalf(err.Error())
-			}
-			log.Printf("Ban DB flushed")
-		},
-	}
-	cmdBan.AddCommand(cmdBanFlush)
-	var cmdBanDelIp = &cobra.Command{
-		Use:     "ip <target>",
-		Short:   "Delete bans for given ip from db",
-		Example: `cscli ban del ip 1.2.3.4`,
-		Args:    cobra.ExactArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			count, err := outputCTX.Delete(args[0])
-			if err != nil {
-				log.Fatalf("failed to delete %s : %v", args[0], err)
-			}
-			log.Infof("Deleted %d entries", count)
-		},
-	}
-	cmdBanDel.AddCommand(cmdBanDelIp)
-	var cmdBanDelRange = &cobra.Command{
-		Use:     "range <target>",
-		Short:   "Delete bans for given ip from db",
-		Example: `cscli ban del range 1.2.3.0/24`,
-		Args:    cobra.ExactArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			count, err := outputCTX.Delete(args[0])
-			if err != nil {
-				log.Fatalf("failed to delete %s : %v", args[0], err)
-			}
-			log.Infof("Deleted %d entries", count)
-		},
-	}
-	cmdBanDel.AddCommand(cmdBanDelRange)
-
-	var cmdBanList = &cobra.Command{
-		Use:   "list",
-		Short: "List local or api bans/remediations",
-		Long: `List the bans, by default only local decisions.
-
-If --all/-a is specified, bans will be displayed without limit (--limit).
-Default limit is 50.
-
-Time can be specified with --at and support a variety of date formats:  
- - Jan  2 15:04:05  
- - Mon Jan 02 15:04:05.000000 2006  
- - 2006-01-02T15:04:05Z07:00  
- - 2006/01/02  
- - 2006/01/02 15:04  
- - 2006-01-02  
- - 2006-01-02 15:04
-`,
-		Example: `ban list --range 0.0.0.0/0 : will list all
-		ban list --country CN
-		ban list --reason crowdsecurity/http-probing
-		ban list --as OVH`,
-		Args: cobra.ExactArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := BanList(); err != nil {
-				log.Fatalf("failed to list bans : %v", err)
-			}
-		},
-	}
-	cmdBanList.PersistentFlags().StringVar(&atTime, "at", "", "List bans at given time")
-	cmdBanList.PersistentFlags().BoolVarP(&displayALL, "all", "a", false, "List bans without limit")
-	cmdBanList.PersistentFlags().BoolVarP(&displayAPI, "api", "", false, "List as well bans received from API")
-	cmdBanList.PersistentFlags().StringVar(&ipFilter, "ip", "", "List bans for given IP")
-	cmdBanList.PersistentFlags().StringVar(&rangeFilter, "range", "", "List bans belonging to given range")
-	cmdBanList.PersistentFlags().StringVar(&reasonFilter, "reason", "", "List bans containing given reason")
-	cmdBanList.PersistentFlags().StringVar(&countryFilter, "country", "", "List bans belonging to given country code")
-	cmdBanList.PersistentFlags().StringVar(&asFilter, "as", "", "List bans belonging to given AS name")
-	cmdBanList.PersistentFlags().IntVar(&displayLimit, "limit", 50, "Limit of bans to display (default 50)")
-
-	cmdBan.AddCommand(cmdBanList)
-	return cmdBan
-}

cmd/crowdsec-cli/bouncers.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"time"
+
+	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
+	"github.com/crowdsecurity/crowdsec/pkg/database"
+	"github.com/enescakir/emoji"
+	"github.com/olekukonko/tablewriter"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var keyName string
+var keyIP string
+var keyLength int
+
+func NewBouncersCmd() *cobra.Command {
+	/* ---- DECISIONS COMMAND */
+	var cmdBouncers = &cobra.Command{
+		Use:   "bouncers [action]",
+		Short: "Manage bouncers",
+		Long: `
+Bouncers Management.
+
+To list/add/delete bouncers
+`,
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				log.Fatalf("unable to create new database client: %s", err)
+			}
+		},
+	}
+
+	var cmdBouncersList = &cobra.Command{
+		Use:     "list",
+		Short:   "List bouncers",
+		Long:    `List bouncers`,
+		Example: `cscli bouncers list`,
+		Args:    cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, arg []string) {
+			blockers, err := dbClient.ListBouncers()
+			if err != nil {
+				log.Errorf("unable to list blockers: %s", err)
+			}
+			if csConfig.Cscli.Output == "human" {
+
+				table := tablewriter.NewWriter(os.Stdout)
+				table.SetCenterSeparator("")
+				table.SetColumnSeparator("")
+
+				table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+				table.SetAlignment(tablewriter.ALIGN_LEFT)
+				table.SetHeader([]string{"Name", "IP Address", "Valid", "Last API pull", "Type", "Version"})
+				for _, b := range blockers {
+					var revoked string
+					if !b.Revoked {
+						revoked = fmt.Sprintf("%s", emoji.CheckMark)
+					} else {
+						revoked = fmt.Sprintf("%s", emoji.Prohibited)
+					}
+					table.Append([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version})
+				}
+				table.Render()
+			} else if csConfig.Cscli.Output == "json" {
+				x, err := json.MarshalIndent(blockers, "", " ")
+				if err != nil {
+					log.Fatalf("failed to unmarshal")
+				}
+				fmt.Printf("%s", string(x))
+			} else if csConfig.Cscli.Output == "raw" {
+				for _, b := range blockers {
+					var revoked string
+					if !b.Revoked {
+						revoked = "validated"
+					} else {
+						revoked = "pending"
+					}
+					fmt.Printf("%s,%s,%s,%s,%s\n", b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Version)
+				}
+			}
+		},
+	}
+	cmdBouncers.AddCommand(cmdBouncersList)
+
+	var cmdBouncersAdd = &cobra.Command{
+		Use:   "add MyBouncerName [--length 16]",
+		Short: "add bouncer",
+		Long:  `add bouncer`,
+		Example: `cscli bouncers add MyBouncerName
+cscli bouncers add MyBouncerName -l 24`,
+		Args: cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, arg []string) {
+			keyName := arg[0]
+			if keyName == "" {
+				log.Errorf("Please provide a name for the api key")
+				return
+			}
+			apiKey, err := middlewares.GenerateAPIKey(keyLength)
+			if err != nil {
+				log.Errorf("unable to generate api key: %s", err)
+				return
+			}
+			err = dbClient.CreateBouncer(keyName, keyIP, middlewares.HashSHA512(apiKey))
+			if err != nil {
+				log.Errorf("unable to create blocker: %s", err)
+				return
+			}
+
+			if csConfig.Cscli.Output == "human" {
+				fmt.Printf("Api key for '%s':\n\n", keyName)
+				fmt.Printf("   %s\n\n", apiKey)
+				fmt.Print("Please keep this key since you will not be able to retrive it!\n")
+			} else if csConfig.Cscli.Output == "raw" {
+				fmt.Printf("%s", apiKey)
+			} else if csConfig.Cscli.Output == "json" {
+				j, err := json.Marshal(apiKey)
+				if err != nil {
+					log.Fatalf("unable to marshal api key")
+				}
+				fmt.Printf("%s", string(j))
+			}
+		},
+	}
+	cmdBouncersAdd.Flags().IntVarP(&keyLength, "length", "l", 16, "length of the api key")
+	cmdBouncers.AddCommand(cmdBouncersAdd)
+
+	var cmdBouncersDelete = &cobra.Command{
+		Use:   "delete MyBouncerName",
+		Short: "delete bouncer",
+		Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, arg []string) {
+			keyName := arg[0]
+			if keyName == "" {
+				log.Errorf("Please provide a bouncer name")
+				return
+			}
+			err := dbClient.DeleteBouncer(keyName)
+			if err != nil {
+				log.Errorf("unable to create blocker: %s", err)
+				return
+			}
+		},
+	}
+	cmdBouncers.AddCommand(cmdBouncersDelete)
+	return cmdBouncers
+}

cmd/crowdsec-cli/capi.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http/httputil"
+	"net/url"
+
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/go-openapi/strfmt"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"gopkg.in/yaml.v2"
+)
+
+var CAPIURLPrefix string = "v2"
+var CAPIBaseURL string = "https://api.crowdsec.net/"
+
+func NewCapiCmd() *cobra.Command {
+	var cmdCapi = &cobra.Command{
+		Use:   "capi [action]",
+		Short: "Manage interaction with Central API (CAPI)",
+		Args:  cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.API.Server == nil {
+				log.Fatalln("There is no API->server configuration")
+			}
+			if csConfig.API.Server.OnlineClient == nil {
+				log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.Self)
+			}
+
+			return nil
+		},
+	}
+
+	var cmdCapiRegister = &cobra.Command{
+		Use:   "register",
+		Short: "Register to Central API (CAPI)",
+		Args:  cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
+			id, err := generateID()
+			if err != nil {
+				log.Fatalf("unable to generate machine id: %s", err)
+			}
+			password := strfmt.Password(generatePassword(passwordLength))
+			apiurl, err := url.Parse(CAPIBaseURL)
+			if err != nil {
+				log.Fatalf("unable to parse api url %s : %s", CAPIBaseURL, err)
+			}
+			_, err = apiclient.RegisterClient(&apiclient.Config{
+				MachineID:     id,
+				Password:      password,
+				UserAgent:     fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
+				URL:           apiurl,
+				VersionPrefix: CAPIURLPrefix,
+			}, nil)
+
+			if err != nil {
+				log.Fatalf("api client register ('%s'): %s", CAPIBaseURL, err)
+			}
+			log.Printf("Successfully registered to Central API (CAPI)")
+
+			var dumpFile string
+
+			if outputFile != "" {
+				dumpFile = outputFile
+			} else if csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
+				dumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
+			} else {
+				dumpFile = ""
+			}
+			apiCfg := csconfig.ApiCredentialsCfg{
+				Login:    id,
+				Password: password.String(),
+				URL:      CAPIBaseURL,
+			}
+			apiConfigDump, err := yaml.Marshal(apiCfg)
+			if err != nil {
+				log.Fatalf("unable to marshal api credentials: %s", err)
+			}
+			if dumpFile != "" {
+				err = ioutil.WriteFile(dumpFile, apiConfigDump, 0600)
+				if err != nil {
+					log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+				}
+				log.Printf("API credentials dumped to '%s'", dumpFile)
+			} else {
+				fmt.Printf("%s\n", string(apiConfigDump))
+			}
+
+			log.Warningf("Run 'systemctl reload crowdsec' for the new configuration to be effective")
+		},
+	}
+	cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
+	cmdCapi.AddCommand(cmdCapiRegister)
+
+	var cmdCapiStatus = &cobra.Command{
+		Use:   "status",
+		Short: "Check status with the Central API (CAPI)",
+		Args:  cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			if csConfig.API.Server == nil {
+				log.Fatalln("There is no configuration on 'api_client:'")
+			}
+			if csConfig.API.Server.OnlineClient == nil {
+				log.Fatalf("Please provide credentials for the API in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+			}
+
+			if csConfig.API.Server.OnlineClient.Credentials == nil {
+				log.Fatalf("no credentials for crowdsec API in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+			}
+
+			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
+			apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
+			if err != nil {
+				log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
+			}
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to load hub index : %s", err)
+			}
+			scenarios, err := cwhub.GetUpstreamInstalledScenariosAsString()
+			if err != nil {
+				log.Fatalf("failed to get scenarios : %s", err.Error())
+			}
+
+			Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), nil)
+			if err != nil {
+				log.Fatalf("init default client: %s", err)
+			}
+			t := models.WatcherAuthRequest{
+				MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
+				Password:  &password,
+				Scenarios: scenarios,
+			}
+			log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+			log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
+			resp, err := Client.Auth.AuthenticateWatcher(context.Background(), t)
+			if err != nil {
+				log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
+			} else {
+				log.Infof("You can successfully interact with Central API (CAPI)")
+			}
+			for k, v := range resp.Response.Header {
+				log.Debugf("[headers] %s : %s", k, v)
+			}
+			dump, _ := httputil.DumpResponse(resp.Response, true)
+			log.Debugf("Response: %s", string(dump))
+		},
+	}
+	cmdCapi.AddCommand(cmdCapiStatus)
+	return cmdCapi
+}

cmd/crowdsec-cli/collections.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+func NewCollectionsCmd() *cobra.Command {
+	var cmdCollections = &cobra.Command{
+		Use:   "collections [action]",
+		Short: "Manage collections from hub",
+		Long:  `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
+		/*TBD fix help*/
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := setHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
+		},
+	}
+
+	var cmdCollectionsInstall = &cobra.Command{
+		Use:     "install collection",
+		Short:   "Install given collection(s)",
+		Long:    `Fetch and install given collection(s) from hub`,
+		Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			for _, name := range args {
+				InstallItem(name, cwhub.COLLECTIONS, forceInstall)
+			}
+		},
+	}
+	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdCollectionsInstall.PersistentFlags().BoolVar(&forceInstall, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdCollections.AddCommand(cmdCollectionsInstall)
+
+	var cmdCollectionsRemove = &cobra.Command{
+		Use:     "remove collection",
+		Short:   "Remove given collection(s)",
+		Long:    `Remove given collection(s) from hub`,
+		Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			if removeAll {
+				RemoveMany(cwhub.COLLECTIONS, "")
+			} else {
+				for _, name := range args {
+					RemoveMany(cwhub.COLLECTIONS, name)
+				}
+			}
+		},
+	}
+	cmdCollectionsRemove.PersistentFlags().BoolVar(&purgeRemove, "purge", false, "Delete source file too")
+	cmdCollectionsRemove.PersistentFlags().BoolVar(&removeAll, "all", false, "Delete all the files in selected scope")
+	cmdCollections.AddCommand(cmdCollectionsRemove)
+
+	var cmdCollectionsUpgrade = &cobra.Command{
+		Use:     "upgrade collection",
+		Short:   "Upgrade given collection(s)",
+		Long:    `Fetch and upgrade given collection(s) from hub`,
+		Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if upgradeAll {
+				UpgradeConfig(cwhub.COLLECTIONS, "", forceUpgrade)
+			} else {
+				for _, name := range args {
+					UpgradeConfig(cwhub.COLLECTIONS, name, forceUpgrade)
+				}
+			}
+		},
+	}
+	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&upgradeAll, "download-only", "d", false, "Only download packages, don't enable")
+	cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceUpgrade, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdCollections.AddCommand(cmdCollectionsUpgrade)
+
+	var cmdCollectionsInspect = &cobra.Command{
+		Use:     "inspect collection",
+		Short:   "Inspect given collection",
+		Long:    `Inspect given collection`,
+		Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			for _, name := range args {
+				InspectItem(name, cwhub.COLLECTIONS)
+			}
+		},
+	}
+	cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "http://127.0.0.1:6060/metrics", "Prometheus url")
+	cmdCollections.AddCommand(cmdCollectionsInspect)
+
+	var cmdCollectionsList = &cobra.Command{
+		Use:     "list collection [-a]",
+		Short:   "List all collections or given one",
+		Long:    `List all collections or given one`,
+		Example: `cscli collections list`,
+		Args:    cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			ListItem(cwhub.COLLECTIONS, args)
+		},
+	}
+	cmdCollectionsList.PersistentFlags().BoolVarP(&listAll, "all", "a", false, "List as well disabled items")
+	cmdCollections.AddCommand(cmdCollectionsList)
+
+	return cmdCollections
+}

cmd/crowdsec-cli/config.go

@@ -1,39 +1,206 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"os"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 )
 
-/*CliCfg is the cli configuration structure, might be unexported*/
-type cliConfig struct {
-	configured          bool
-	ConfigFilePath      string `yaml:"config_file"`
-	configFolder        string
-	output              string
-	HubFolder           string `yaml:"hub_folder"`
-	InstallFolder       string
-	BackendPluginFolder string `yaml:"backend_folder"`
-	DataFolder          string `yaml:"data_folder"`
-	SimulationCfgPath   string `yaml:"simulation_path,omitempty"`
-	SimulationCfg       *csconfig.SimulationConfig
+type OldAPICfg struct {
+	MachineID string `json:"machine_id"`
+	Password  string `json:"password"`
+}
+
+/* Backup crowdsec configurations to directory <dirPath> :
+
+- Main config (config.yaml)
+- Profiles config (profiles.yaml)
+- Simulation config (simulation.yaml)
+- Backup of API credentials (local API and online API)
+- List of scenarios, parsers, postoverflows and collections that are up-to-date
+- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+*/
+func backupConfigToDirectory(dirPath string) error {
+	var err error
+
+	if dirPath == "" {
+		return fmt.Errorf("directory path can't be empty")
+	}
+	log.Infof("Starting configuration backup")
+	_, err = os.Stat(dirPath)
+	if err == nil {
+		return fmt.Errorf("%s already exists", dirPath)
+	}
+	if err = os.MkdirAll(dirPath, os.ModePerm); err != nil {
+		return fmt.Errorf("error while creating %s : %s", dirPath, err)
+	}
+
+	if csConfig.ConfigPaths.SimulationFilePath != "" {
+		backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath)
+		if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation, err)
+		}
+		log.Infof("Saved simulation to %s", backupSimulation)
+	}
+	if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" {
+		backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath)
+		if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
+		}
+		log.Infof("Saved acquis to %s", backupAcquisition)
+	}
+	if ConfigFilePath != "" {
+		backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
+		if err = types.CopyFile(ConfigFilePath, backupMain); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err)
+		}
+		log.Infof("Saved default yaml to %s", backupMain)
+	}
+	if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
+		backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
+		if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
+		}
+		log.Infof("Saved online API credentials to %s", backupCAPICreds)
+	}
+	if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
+		backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
+		if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
+		}
+		log.Infof("Saved local API credentials to %s", backupLAPICreds)
+	}
+	if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" {
+		backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
+		if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
+		}
+		log.Infof("Saved profiles to %s", backupProfiles)
+	}
+
+	if err = BackupHub(dirPath); err != nil {
+		return fmt.Errorf("failed to backup hub config : %s", err)
+	}
+
+	return nil
+}
+
+/* Restore crowdsec configurations to directory <dirPath> :
+
+- Main config (config.yaml)
+- Profiles config (profiles.yaml)
+- Simulation config (simulation.yaml)
+- Backup of API credentials (local API and online API)
+- List of scenarios, parsers, postoverflows and collections that are up-to-date
+- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+*/
+func restoreConfigFromDirectory(dirPath string) error {
+	var err error
+
+	if !restoreOldBackup {
+		backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
+		if _, err = os.Stat(backupMain); err == nil {
+			if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" {
+				if err = types.CopyFile(backupMain, csConfig.ConfigPaths.ConfigDir); err != nil {
+					return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err)
+				}
+			}
+		}
+
+		// Now we have config.yaml, we should regenerate config struct to have rights paths etc
+		ConfigFilePath = fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)
+		initConfig()
+
+		backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
+		if _, err = os.Stat(backupCAPICreds); err == nil {
+			if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err)
+			}
+		}
+
+		backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
+		if _, err = os.Stat(backupLAPICreds); err == nil {
+			if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err)
+			}
+		}
+
+		backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
+		if _, err = os.Stat(backupProfiles); err == nil {
+			if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err)
+			}
+		}
+	} else {
+		var oldAPICfg OldAPICfg
+		backupOldAPICfg := fmt.Sprintf("%s/api_creds.json", dirPath)
+
+		jsonFile, err := os.Open(backupOldAPICfg)
+		if err != nil {
+			log.Warningf("failed to open %s : %s", backupOldAPICfg, err)
+		} else {
+			byteValue, _ := ioutil.ReadAll(jsonFile)
+			err = json.Unmarshal(byteValue, &oldAPICfg)
+			if err != nil {
+				return fmt.Errorf("failed to load json file %s : %s", backupOldAPICfg, err)
+			}
+
+			apiCfg := csconfig.ApiCredentialsCfg{
+				Login:    oldAPICfg.MachineID,
+				Password: oldAPICfg.Password,
+				URL:      CAPIBaseURL,
+			}
+			apiConfigDump, err := yaml.Marshal(apiCfg)
+			if err != nil {
+				return fmt.Errorf("unable to dump api credentials: %s", err)
+			}
+			apiConfigDumpFile := fmt.Sprintf("%s/online_api_credentials.yaml", csConfig.ConfigPaths.ConfigDir)
+			if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
+				apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
+			}
+			err = ioutil.WriteFile(apiConfigDumpFile, apiConfigDump, 0644)
+			if err != nil {
+				return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
+			}
+			log.Infof("Saved API credentials to %s", apiConfigDumpFile)
+		}
+	}
+
+	backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath)
+	if _, err = os.Stat(backupSimulation); err == nil {
+		if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err)
+		}
+	}
+
+	backupAcquisition := fmt.Sprintf("%s/acquis.yaml", dirPath)
+	if _, err = os.Stat(backupAcquisition); err == nil {
+		if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err)
+		}
+	}
+
+	if err = RestoreHub(dirPath); err != nil {
+		return fmt.Errorf("failed to restore hub config : %s", err)
+	}
+
+	return nil
 }
 
 func NewConfigCmd() *cobra.Command {
 
 	var cmdConfig = &cobra.Command{
-		Use:   "config [command] <value>",
-		Short: "Allows to view/edit cscli config",
-		Long: `Allow to configure database plugin path and installation directory.
-If no commands are specified, config is in interactive mode.`,
-		Example: `- cscli config show
-- cscli config prompt`,
-		Args: cobra.ExactArgs(1),
+		Use:   "config [command]",
+		Short: "Allows to view current config",
+		Args:  cobra.ExactArgs(0),
 	}
 	var cmdConfigShow = &cobra.Command{
 		Use:   "show",
@@ -41,21 +208,128 @@ If no commands are specified, config is in interactive mode.`,
 		Long:  `Displays the current cli configuration.`,
 		Args:  cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
-			if config.output == "json" {
-				log.WithFields(log.Fields{
-					"crowdsec_configuration_file": config.ConfigFilePath,
-					"backend_folder":              config.BackendPluginFolder,
-					"data_folder":                 config.DataFolder,
-				}).Warning("Current config")
-			} else {
-				x, err := yaml.Marshal(config)
+			switch csConfig.Cscli.Output {
+			case "human":
+				fmt.Printf("Global:\n")
+				fmt.Printf("   - Configuration Folder   : %s\n", csConfig.ConfigPaths.ConfigDir)
+				fmt.Printf("   - Data Folder            : %s\n", csConfig.ConfigPaths.DataDir)
+				fmt.Printf("   - Log Folder             : %s\n", csConfig.Common.LogDir)
+				fmt.Printf("   - Hub Folder             : %s\n", csConfig.ConfigPaths.HubDir)
+				fmt.Printf("   - Simulation File        : %s\n", csConfig.ConfigPaths.SimulationFilePath)
+				fmt.Printf("   - Log level              : %s\n", csConfig.Common.LogLevel)
+				fmt.Printf("   - Log Media              : %s\n", csConfig.Common.LogMedia)
+				fmt.Printf("Crowdsec:\n")
+				fmt.Printf("  - Acquisition File        : %s\n", csConfig.Crowdsec.AcquisitionFilePath)
+				fmt.Printf("  - Parsers routines        : %d\n", csConfig.Crowdsec.ParserRoutinesCount)
+				fmt.Printf("cscli:\n")
+				fmt.Printf("  - Output                  : %s\n", csConfig.Cscli.Output)
+				fmt.Printf("  - Hub Branch              : %s\n", csConfig.Cscli.HubBranch)
+				fmt.Printf("  - Hub Folder              : %s\n", csConfig.Cscli.HubDir)
+				fmt.Printf("API Client:\n")
+				fmt.Printf("  - URL                     : %s\n", csConfig.API.Client.Credentials.URL)
+				fmt.Printf("  - Login                   : %s\n", csConfig.API.Client.Credentials.Login)
+				fmt.Printf("  - Credentials File        : %s\n", csConfig.API.Client.CredentialsFilePath)
+				fmt.Printf("Local API Server:\n")
+				fmt.Printf("  - Listen URL              : %s\n", csConfig.API.Server.ListenURI)
+				fmt.Printf("  - Profile File            : %s\n", csConfig.API.Server.ProfilesPath)
+				if csConfig.API.Server.TLS != nil {
+					if csConfig.API.Server.TLS.CertFilePath != "" {
+						fmt.Printf("  - Cert File : %s\n", csConfig.API.Server.TLS.CertFilePath)
+					}
+					if csConfig.API.Server.TLS.KeyFilePath != "" {
+						fmt.Printf("  - Key File  : %s\n", csConfig.API.Server.TLS.KeyFilePath)
+					}
+				}
+				fmt.Printf("  - Database:\n")
+				fmt.Printf("      - Type                : %s\n", csConfig.DbConfig.Type)
+				switch csConfig.DbConfig.Type {
+				case "sqlite":
+					fmt.Printf("      - Path                : %s\n", csConfig.DbConfig.DbPath)
+				case "mysql", "postgresql", "postgres":
+					fmt.Printf("      - Host                : %s\n", csConfig.DbConfig.Host)
+					fmt.Printf("      - Port                : %d\n", csConfig.DbConfig.Port)
+					fmt.Printf("      - User                : %s\n", csConfig.DbConfig.User)
+					fmt.Printf("      - DB Name             : %s\n", csConfig.DbConfig.DbName)
+				}
+				if csConfig.DbConfig.Flush != nil {
+					if *csConfig.DbConfig.Flush.MaxAge != "" {
+						fmt.Printf("      - Flush age           : %s\n", *csConfig.DbConfig.Flush.MaxAge)
+					}
+					if *csConfig.DbConfig.Flush.MaxItems != 0 {
+						fmt.Printf("      - Flush size          : %d\n", *csConfig.DbConfig.Flush.MaxItems)
+					}
+				}
+
+				fmt.Printf("Central API:\n")
+				fmt.Printf("  - URL                     : %s\n", csConfig.API.Server.OnlineClient.Credentials.URL)
+				fmt.Printf("  - Login                   : %s\n", csConfig.API.Server.OnlineClient.Credentials.Login)
+				fmt.Printf("  - Credentials File        : %s\n", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+			case "json":
+				data, err := json.MarshalIndent(csConfig, "", "  ")
 				if err != nil {
-					log.Fatalf("failed to marshal current configuration : %v", err)
+					log.Fatalf("failed to marshal configuration: %s", err)
 				}
-				fmt.Printf("%s", x)
+				fmt.Printf("%s\n", string(data))
+			case "raw":
+				data, err := yaml.Marshal(csConfig)
+				if err != nil {
+					log.Fatalf("failed to marshal configuration: %s", err)
+				}
+				fmt.Printf("%s\n", string(data))
 			}
 		},
 	}
 	cmdConfig.AddCommand(cmdConfigShow)
+
+	var cmdConfigBackup = &cobra.Command{
+		Use:   "backup <directory>",
+		Short: "Backup current config",
+		Long: `Backup the current crowdsec configuration including :
+
+- Main config (config.yaml)
+- Simulation config (simulation.yaml)
+- Profiles config (profiles.yaml)
+- List of scenarios, parsers, postoverflows and collections that are up-to-date
+- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Backup of API credentials (local API and online API)`,
+		Example: `cscli config backup ./my-backup`,
+		Args:    cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			if err = cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			if err = backupConfigToDirectory(args[0]); err != nil {
+				log.Fatalf("Failed to backup configurations: %s", err)
+			}
+		},
+	}
+	cmdConfig.AddCommand(cmdConfigBackup)
+
+	var cmdConfigRestore = &cobra.Command{
+		Use:   "restore <directory>",
+		Short: "Restore config in backup <directory>",
+		Long: `Restore the crowdsec configuration from specified backup <directory> including:
+
+- Main config (config.yaml)
+- Simulation config (simulation.yaml)
+- Profiles config (profiles.yaml)
+- List of scenarios, parsers, postoverflows and collections that are up-to-date
+- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Backup of API credentials (local API and online API)`,
+		Args: cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			if err = cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			if err := restoreConfigFromDirectory(args[0]); err != nil {
+				log.Fatalf("failed restoring configurations from %s : %s", args[0], err)
+			}
+		},
+	}
+	cmdConfigRestore.PersistentFlags().BoolVar(&restoreOldBackup, "old-backup", false, "To use when you are upgrading crowdsec v0.X to v1.X and you need to restore backup from v0.X")
+	cmdConfig.AddCommand(cmdConfigRestore)
+
 	return cmdConfig
 }

cmd/crowdsec-cli/dashboard.go

@@ -1,59 +1,56 @@
 package main
 
 import (
-	"archive/zip"
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
 	"os"
-	"path"
-	"strings"
-	"time"
+	"path/filepath"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/crowdsecurity/crowdsec/pkg/metabase"
 
-	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
-	"github.com/dghubble/sling"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/api/types/mount"
-	"github.com/docker/docker/client"
-	"github.com/docker/go-connections/nat"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 var (
-	metabaseImage  = "metabase/metabase"
-	metabaseDbURI  = "https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/metabase.db.zip"
-	metabaseDbPath = "/var/lib/crowdsec/data"
+	metabaseUser         = "crowdsec@crowdsec.net"
+	metabasePassword     string
+	metabaseDbPath       string
+	metabaseConfigPath   string
+	metabaseConfigFolder = "metabase/"
+	metabaseConfigFile   = "metabase.yaml"
+	metabaseImage        = "metabase/metabase"
 	/**/
 	metabaseListenAddress = "127.0.0.1"
 	metabaseListenPort    = "3000"
 	metabaseContainerID   = "/crowdsec-metabase"
+
+	forceYes bool
+
+	dockerGatewayIPAddr = "172.17.0.1"
 	/*informations needed to setup a random password on user's behalf*/
-	metabaseURI          = "http://localhost:3000/api/"
-	metabaseURISession   = "session"
-	metabaseURIRescan    = "database/2/rescan_values"
-	metabaseURIUpdatepwd = "user/1/password"
-	defaultPassword      = "c6cmetabase"
-	defaultEmail         = "metabase@crowdsec.net"
 )
 
 func NewDashboardCmd() *cobra.Command {
 	/* ---- UPDATE COMMAND */
 	var cmdDashboard = &cobra.Command{
-		Use:   "dashboard",
-		Short: "Start a dashboard (metabase) container.",
-		Long:  `Start a metabase container exposing dashboards and metrics.`,
+		Use:   "dashboard [command]",
+		Short: "Manage your metabase dashboard container",
+		Long:  `Install/Start/Stop/Remove a metabase container exposing dashboard and metrics.`,
 		Args:  cobra.ExactArgs(1),
-		Example: `cscli dashboard setup
+		Example: `
+cscli dashboard setup
 cscli dashboard start
 cscli dashboard stop
-cscli dashboard setup --force`,
+cscli dashboard remove
+`,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			metabaseConfigFolderPath := filepath.Join(csConfig.ConfigPaths.ConfigDir, metabaseConfigFolder)
+			metabaseConfigPath = filepath.Join(metabaseConfigFolderPath, metabaseConfigFile)
+			if err := os.MkdirAll(metabaseConfigFolderPath, os.ModePerm); err != nil {
+				log.Fatalf(err.Error())
+			}
+		},
 	}
 
 	var force bool
@@ -62,33 +59,42 @@ cscli dashboard setup --force`,
 		Short: "Setup a metabase container.",
 		Long:  `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
 		Args:  cobra.ExactArgs(0),
-		Example: `cscli dashboard setup
-cscli dashboard setup --force
-cscli dashboard setup -l 0.0.0.0 -p 443
+		Example: `
+cscli dashboard setup
+cscli dashboard setup --listen 0.0.0.0
+cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
  `,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := downloadMetabaseDB(force); err != nil {
-				log.Fatalf("Failed to download metabase DB : %s", err)
+			if metabaseDbPath == "" {
+				metabaseDbPath = csConfig.ConfigPaths.DataDir
 			}
-			log.Infof("Downloaded metabase DB")
-			if err := createMetabase(); err != nil {
-				log.Fatalf("Failed to start metabase container : %s", err)
+
+			if metabasePassword == "" {
+				metabasePassword = generatePassword(16)
 			}
-			log.Infof("Started metabase")
-			newpassword := generatePassword(64)
-			if err := resetMetabasePassword(newpassword); err != nil {
-				log.Fatalf("Failed to reset password : %s", err)
+			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath)
+			if err != nil {
+				log.Fatalf(err.Error())
 			}
-			log.Infof("Setup finished")
-			log.Infof("url : http://%s:%s", metabaseListenAddress, metabaseListenPort)
-			log.Infof("username: %s", defaultEmail)
-			log.Infof("password: %s", newpassword)
+
+			if err := mb.DumpConfig(metabaseConfigPath); err != nil {
+				log.Fatalf(err.Error())
+			}
+
+			log.Infof("Metabase is ready")
+			fmt.Println()
+			fmt.Printf("\tURL       : '%s'\n", mb.Config.ListenURL)
+			fmt.Printf("\tusername  : '%s'\n", mb.Config.Username)
+			fmt.Printf("\tpassword  : '%s'\n", mb.Config.Password)
 		},
 	}
 	cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files.")
-	cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", metabaseDbPath, "Shared directory with metabase container.")
+	cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container.")
 	cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
 	cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
+	//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
+	cmdDashSetup.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
+
 	cmdDashboard.AddCommand(cmdDashSetup)
 
 	var cmdDashStart = &cobra.Command{
@@ -97,7 +103,11 @@ cscli dashboard setup -l 0.0.0.0 -p 443
 		Long:  `Stats the metabase container using docker.`,
 		Args:  cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := startMetabase(); err != nil {
+			mb, err := metabase.NewMetabase(metabaseConfigPath)
+			if err != nil {
+				log.Fatalf(err.Error())
+			}
+			if err := mb.Container.Start(); err != nil {
 				log.Fatalf("Failed to start metabase container : %s", err)
 			}
 			log.Infof("Started metabase")
@@ -106,270 +116,69 @@ cscli dashboard setup -l 0.0.0.0 -p 443
 	}
 	cmdDashboard.AddCommand(cmdDashStart)
 
-	var remove bool
 	var cmdDashStop = &cobra.Command{
 		Use:   "stop",
 		Short: "Stops the metabase container.",
 		Long:  `Stops the metabase container using docker.`,
 		Args:  cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := stopMetabase(remove); err != nil {
-				log.Fatalf("Failed to stop metabase container : %s", err)
+			if err := metabase.StopContainer(metabaseContainerID); err != nil {
+				log.Fatalf("unable to stop container '%s': %s", metabaseContainerID, err)
 			}
 		},
 	}
-	cmdDashStop.Flags().BoolVarP(&remove, "remove", "r", false, "remove (docker rm) container as well.")
 	cmdDashboard.AddCommand(cmdDashStop)
-	return cmdDashboard
-}
-
-func downloadMetabaseDB(force bool) error {
-
-	metabaseDBSubpath := path.Join(metabaseDbPath, "metabase.db")
-
-	_, err := os.Stat(metabaseDBSubpath)
-	if err == nil && !force {
-		log.Printf("%s exists, skip.", metabaseDBSubpath)
-		return nil
-	}
-
-	if err := os.MkdirAll(metabaseDBSubpath, 0755); err != nil {
-		return fmt.Errorf("failed to create %s : %s", metabaseDBSubpath, err)
-	}
-
-	req, err := http.NewRequest("GET", metabaseDbURI, nil)
-	if err != nil {
-		return fmt.Errorf("failed to build request to fetch metabase db : %s", err)
-	}
-	//This needs to be removed once we move the zip out of github
-	req.Header.Add("Accept", `application/vnd.github.v3.raw`)
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed request to fetch metabase db : %s", err)
-	}
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("got http %d while requesting metabase db %s, stop", resp.StatusCode, metabaseDbURI)
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("failed request read while fetching metabase db : %s", err)
-	}
-
-	log.Printf("Got %d bytes archive", len(body))
-	if err := extractMetabaseDB(bytes.NewReader(body)); err != nil {
-		return fmt.Errorf("while extracting zip : %s", err)
-	}
-	return nil
-}
-
-func extractMetabaseDB(buf *bytes.Reader) error {
-	r, err := zip.NewReader(buf, int64(buf.Len()))
-	if err != nil {
-		log.Fatal(err)
-	}
-	for _, f := range r.File {
-		if strings.Contains(f.Name, "..") {
-			return fmt.Errorf("invalid path '%s' in archive", f.Name)
-		}
-		tfname := fmt.Sprintf("%s/%s", metabaseDbPath, f.Name)
-		log.Debugf("%s -> %d", f.Name, f.UncompressedSize64)
-		if f.UncompressedSize64 == 0 {
-			continue
-		}
-		tfd, err := os.OpenFile(tfname, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644)
-		if err != nil {
-			return fmt.Errorf("failed opening target file '%s' : %s", tfname, err)
-		}
-		rc, err := f.Open()
-		if err != nil {
-			return fmt.Errorf("while opening zip content %s : %s", f.Name, err)
-		}
-		written, err := io.Copy(tfd, rc)
-		if err == io.EOF {
-			log.Printf("files finished ok")
-		} else if err != nil {
-			return fmt.Errorf("while copying content to %s : %s", tfname, err)
-		}
-		log.Infof("written %d bytes to %s", written, tfname)
-		rc.Close()
-	}
-	return nil
-}
-
-func resetMetabasePassword(newpassword string) error {
-
-	httpctx := sling.New().Base(metabaseURI).Set("User-Agent", fmt.Sprintf("Crowdsec/%s", cwversion.VersionStr()))
-
-	log.Printf("Waiting for metabase API to be up (can take up to a minute)")
-	for {
-		sessionreq, err := httpctx.New().Post(metabaseURISession).BodyJSON(map[string]string{"username": defaultEmail, "password": defaultPassword}).Request()
-		if err != nil {
-			return fmt.Errorf("api signin: HTTP request creation failed: %s", err)
-		}
-		httpClient := http.Client{Timeout: 20 * time.Second}
-		resp, err := httpClient.Do(sessionreq)
-		if err != nil {
-			fmt.Printf(".")
-			log.Debugf("While waiting for metabase to be up : %s", err)
-			time.Sleep(1 * time.Second)
-			continue
-		}
-		defer resp.Body.Close()
-		fmt.Printf("\n")
-		log.Printf("Metabase API is up")
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return fmt.Errorf("metabase session unable to read API response body: '%s'", err)
-		}
-		if resp.StatusCode != 200 {
-			return fmt.Errorf("metabase session http error (%d): %s", resp.StatusCode, string(body))
-		}
-		log.Printf("Successfully authenticated")
-		jsonResp := make(map[string]string)
-		err = json.Unmarshal(body, &jsonResp)
-		if err != nil {
-			return fmt.Errorf("failed to unmarshal metabase api response '%s': %s", string(body), err.Error())
-		}
-		log.Debugf("unmarshaled response : %v", jsonResp)
-		httpctx = httpctx.Set("Cookie", fmt.Sprintf("metabase.SESSION=%s", jsonResp["id"]))
-		break
-	}
 
-	/*rescan values*/
-	sessionreq, err := httpctx.New().Post(metabaseURIRescan).Request()
-	if err != nil {
-		return fmt.Errorf("metabase rescan_values http error : %s", err)
-	}
-	httpClient := http.Client{Timeout: 20 * time.Second}
-	resp, err := httpClient.Do(sessionreq)
-	if err != nil {
-		return fmt.Errorf("while trying to do rescan api call to metabase : %s", err)
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("while reading rescan api call response : %s", err)
-	}
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("got '%s' (http:%d) while trying to rescan metabase", string(body), resp.StatusCode)
-	}
-	/*update password*/
-	sessionreq, err = httpctx.New().Put(metabaseURIUpdatepwd).BodyJSON(map[string]string{
-		"id":           "1",
-		"password":     newpassword,
-		"old_password": defaultPassword}).Request()
-	if err != nil {
-		return fmt.Errorf("metabase password change http error : %s", err)
-	}
-	httpClient = http.Client{Timeout: 20 * time.Second}
-	resp, err = httpClient.Do(sessionreq)
-	if err != nil {
-		return fmt.Errorf("while trying to reset metabase password : %s", err)
-	}
-	defer resp.Body.Close()
-	body, err = ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("while reading from %s: '%s'", metabaseURIUpdatepwd, err)
-	}
-	if resp.StatusCode != 200 {
-		log.Printf("Got %s (http:%d) while trying to reset password.", string(body), resp.StatusCode)
-		log.Printf("Password has probably already been changed.")
-		log.Printf("Use the dashboard install command to reset existing setup.")
-		return fmt.Errorf("got http error %d on %s : %s", resp.StatusCode, metabaseURIUpdatepwd, string(body))
-	}
-	log.Printf("Changed password !")
-	return nil
-}
-
-func startMetabase() error {
-	ctx := context.Background()
-	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		return fmt.Errorf("failed to create docker client : %s", err)
-	}
-
-	if err := cli.ContainerStart(ctx, metabaseContainerID, types.ContainerStartOptions{}); err != nil {
-		return fmt.Errorf("failed while starting %s : %s", metabaseContainerID, err)
-	}
-
-	return nil
-}
-
-func stopMetabase(remove bool) error {
-	log.Printf("Stop docker metabase %s", metabaseContainerID)
-	ctx := context.Background()
-	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		return fmt.Errorf("failed to create docker client : %s", err)
-	}
-	var to time.Duration = 20 * time.Second
-	if err := cli.ContainerStop(ctx, metabaseContainerID, &to); err != nil {
-		return fmt.Errorf("failed while stopping %s : %s", metabaseContainerID, err)
-	}
-
-	if remove {
-		log.Printf("Removing docker metabase %s", metabaseContainerID)
-		if err := cli.ContainerRemove(ctx, metabaseContainerID, types.ContainerRemoveOptions{}); err != nil {
-			return fmt.Errorf("failed remove container %s : %s", metabaseContainerID, err)
-		}
-	}
-	return nil
-}
-
-func createMetabase() error {
-	ctx := context.Background()
-	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
-	if err != nil {
-		return fmt.Errorf("failed to start docker client : %s", err)
-	}
-
-	log.Printf("Pulling docker image %s", metabaseImage)
-	reader, err := cli.ImagePull(ctx, metabaseImage, types.ImagePullOptions{})
-	if err != nil {
-		return fmt.Errorf("failed to pull docker image : %s", err)
-	}
-	defer reader.Close()
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		fmt.Print(".")
-	}
-	if err := scanner.Err(); err != nil {
-		return fmt.Errorf("failed to read imagepull reader: %s", err)
-	}
-	fmt.Print("\n")
+	var cmdDashRemove = &cobra.Command{
+		Use:   "remove",
+		Short: "removes the metabase container.",
+		Long:  `removes the metabase container using docker.`,
+		Args:  cobra.ExactArgs(0),
+		Example: `
+cscli dashboard remove
+cscli dashboard remove --force
+ `,
+		Run: func(cmd *cobra.Command, args []string) {
+			answer := true
+			if !forceYes {
+				prompt := &survey.Confirm{
+					Message: "Do you really want to remove crowdsec dashboard? (all your changes will be lost)",
+					Default: true,
+				}
+				if err := survey.AskOne(prompt, &answer); err != nil {
+					log.Fatalf("unable to ask to force: %s", err)
+				}
+			}
 
-	hostConfig := &container.HostConfig{
-		PortBindings: nat.PortMap{
-			"3000/tcp": []nat.PortBinding{
-				{
-					HostIP:   metabaseListenAddress,
-					HostPort: metabaseListenPort,
-				},
-			},
-		},
-		Mounts: []mount.Mount{
-			{
-				Type:   mount.TypeBind,
-				Source: metabaseDbPath,
-				Target: "/metabase-data",
-			},
+			if answer {
+				if metabase.IsContainerExist(metabaseContainerID) {
+					log.Debugf("Stopping container %s", metabaseContainerID)
+					if err := metabase.StopContainer(metabaseContainerID); err != nil {
+						log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
+					}
+					log.Debugf("Removing container %s", metabaseContainerID)
+					if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
+						log.Warningf("unable to remove container '%s': %s", metabaseContainerID, err)
+					}
+					log.Infof("container %s stopped & removed", metabaseContainerID)
+				}
+				log.Debugf("Removing database %s", csConfig.ConfigPaths.DataDir)
+				if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
+					log.Warningf("failed to remove metabase internal db : %s", err)
+				}
+				if force {
+					log.Debugf("Removing image %s", metabaseImage)
+					if err := metabase.RemoveImageContainer(metabaseImage); err != nil {
+            log.Warningf("Failed to remove metabase container %s : %s", metabaseImage, err)
+
+					}
+				}
+			}
 		},
 	}
-	dockerConfig := &container.Config{
-		Image: metabaseImage,
-		Tty:   true,
-		Env:   []string{"MB_DB_FILE=/metabase-data/metabase.db"},
-	}
+	cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Force remove : stop the container if running and remove.")
+	cmdDashRemove.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
+	cmdDashboard.AddCommand(cmdDashRemove)
 
-	log.Printf("Creating container")
-	resp, err := cli.ContainerCreate(ctx, dockerConfig, hostConfig, nil, metabaseContainerID)
-	if err != nil {
-		return fmt.Errorf("failed to create container : %s", err)
-	}
-	log.Printf("Starting container")
-	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
-		return fmt.Errorf("failed to start docker container : %s", err)
-	}
-	return nil
+	return cmdDashboard
 }

cmd/crowdsec-cli/decisions.go

@@ -0,0 +1,443 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/database"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/go-openapi/strfmt"
+	"github.com/olekukonko/tablewriter"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+var Client *apiclient.ApiClient
+
+func DecisionsToTable(alerts *models.GetAlertsResponse) error {
+	/*here we cheat a bit : to make it more readable for the user, we dedup some entries*/
+	var spamLimit map[string]bool = make(map[string]bool)
+
+	/*process in reverse order to keep the latest item only*/
+	for aIdx := len(*alerts) - 1; aIdx >= 0; aIdx-- {
+		alertItem := (*alerts)[aIdx]
+		newDecisions := make([]*models.Decision, 0)
+		for _, decisionItem := range alertItem.Decisions {
+			spamKey := fmt.Sprintf("%t:%s:%s:%s", *decisionItem.Simulated, *decisionItem.Type, *decisionItem.Scope, *decisionItem.Value)
+			if _, ok := spamLimit[spamKey]; ok {
+				continue
+			}
+			spamLimit[spamKey] = true
+			newDecisions = append(newDecisions, decisionItem)
+		}
+		alertItem.Decisions = newDecisions
+	}
+	if csConfig.Cscli.Output == "raw" {
+		fmt.Printf("id,source,ip,reason,action,country,as,events_count,expiration,simulated,alert_id\n")
+		for _, alertItem := range *alerts {
+			for _, decisionItem := range alertItem.Decisions {
+				fmt.Printf("%v,%v,%v,%v,%v,%v,%v,%v,%v,%v,%v\n",
+					decisionItem.ID,
+					*decisionItem.Origin,
+					*decisionItem.Scope+":"+*decisionItem.Value,
+					*decisionItem.Scenario,
+					*decisionItem.Type,
+					alertItem.Source.Cn,
+					alertItem.Source.AsNumber+" "+alertItem.Source.AsName,
+					*alertItem.EventsCount,
+					*decisionItem.Duration,
+					*decisionItem.Simulated,
+					alertItem.ID)
+			}
+		}
+	} else if csConfig.Cscli.Output == "json" {
+		x, _ := json.MarshalIndent(alerts, "", " ")
+		fmt.Printf("%s", string(x))
+	} else if csConfig.Cscli.Output == "human" {
+
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetHeader([]string{"ID", "Source", "Scope:Value", "Reason", "Action", "Country", "AS", "Events", "expiration", "Alert ID"})
+
+		if len(*alerts) == 0 {
+			fmt.Println("No active decisions")
+			return nil
+		}
+
+		for _, alertItem := range *alerts {
+			for _, decisionItem := range alertItem.Decisions {
+				if *alertItem.Simulated {
+					*decisionItem.Type = fmt.Sprintf("(simul)%s", *decisionItem.Type)
+				}
+				table.Append([]string{
+					strconv.Itoa(int(decisionItem.ID)),
+					*decisionItem.Origin,
+					*decisionItem.Scope + ":" + *decisionItem.Value,
+					*decisionItem.Scenario,
+					*decisionItem.Type,
+					alertItem.Source.Cn,
+					alertItem.Source.AsNumber + " " + alertItem.Source.AsName,
+					strconv.Itoa(int(*alertItem.EventsCount)),
+					*decisionItem.Duration,
+					strconv.Itoa(int(alertItem.ID)),
+				})
+			}
+		}
+		table.Render() // Send output
+	}
+	return nil
+}
+
+func NewDecisionsCmd() *cobra.Command {
+	/* ---- DECISIONS COMMAND */
+	var cmdDecisions = &cobra.Command{
+		Use:     "decisions [action]",
+		Short:   "Manage decisions",
+		Long:    `Add/List/Delete decisions from LAPI`,
+		Example: `cscli decisions [action] [filter]`,
+		/*TBD example*/
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			if csConfig.API.Client == nil {
+				log.Fatalln("There is no configuration on 'api_client:'")
+			}
+			if csConfig.API.Client.Credentials == nil {
+				log.Fatalf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath)
+			}
+			password := strfmt.Password(csConfig.API.Client.Credentials.Password)
+			apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
+			if err != nil {
+				log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Client.Credentials.URL, err)
+			}
+			Client, err = apiclient.NewClient(&apiclient.Config{
+				MachineID:     csConfig.API.Client.Credentials.Login,
+				Password:      password,
+				UserAgent:     fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
+				URL:           apiurl,
+				VersionPrefix: "v1",
+			})
+			if err != nil {
+				log.Fatalf("creating api client : %s", err)
+			}
+		},
+	}
+
+	var filter = apiclient.AlertsListOpts{
+		ValueEquals:    new(string),
+		ScopeEquals:    new(string),
+		ScenarioEquals: new(string),
+		IPEquals:       new(string),
+		RangeEquals:    new(string),
+		Since:          new(string),
+		Until:          new(string),
+		TypeEquals:     new(string),
+		IncludeCAPI:    new(bool),
+	}
+	NoSimu := new(bool)
+	var cmdDecisionsList = &cobra.Command{
+		Use:   "list [options]",
+		Short: "List decisions from LAPI",
+		Example: `cscli decisions list -i 1.2.3.4
+cscli decisions list -r 1.2.3.0/24
+cscli decisions list -s crowdsecurity/ssh-bf
+cscli decisions list -t ban
+`,
+		Args: cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			/*take care of shorthand options*/
+			if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
+				log.Fatalf("%s", err)
+			}
+			filter.ActiveDecisionEquals = new(bool)
+			*filter.ActiveDecisionEquals = true
+			if NoSimu != nil && *NoSimu {
+				*filter.IncludeSimulated = false
+			}
+			/*nulify the empty entries to avoid bad filter*/
+			if *filter.Until == "" {
+				filter.Until = nil
+			} else {
+				/*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/
+				if strings.HasSuffix(*filter.Until, "d") {
+					realDuration := strings.TrimSuffix(*filter.Until, "d")
+					days, err := strconv.Atoi(realDuration)
+					if err != nil {
+						cmd.Help()
+						log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
+					}
+					*filter.Until = fmt.Sprintf("%d%s", days*24, "h")
+				}
+			}
+			if *filter.Since == "" {
+				filter.Since = nil
+			} else {
+				/*time.ParseDuration support hours 'h' as bigger unit, let's make the user's life easier*/
+				if strings.HasSuffix(*filter.Since, "d") {
+					realDuration := strings.TrimSuffix(*filter.Since, "d")
+					days, err := strconv.Atoi(realDuration)
+					if err != nil {
+						cmd.Help()
+						log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
+					}
+					*filter.Since = fmt.Sprintf("%d%s", days*24, "h")
+				}
+			}
+			if *filter.TypeEquals == "" {
+				filter.TypeEquals = nil
+			}
+			if *filter.ValueEquals == "" {
+				filter.ValueEquals = nil
+			}
+			if *filter.ScopeEquals == "" {
+				filter.ScopeEquals = nil
+			}
+			if *filter.ScenarioEquals == "" {
+				filter.ScenarioEquals = nil
+			}
+			if *filter.IPEquals == "" {
+				filter.IPEquals = nil
+			}
+			if *filter.RangeEquals == "" {
+				filter.RangeEquals = nil
+			}
+			alerts, _, err := Client.Alerts.List(context.Background(), filter)
+			if err != nil {
+				log.Fatalf("Unable to list decisions : %v", err.Error())
+			}
+
+			err = DecisionsToTable(alerts)
+			if err != nil {
+				log.Fatalf("unable to list decisions : %v", err.Error())
+			}
+		},
+	}
+	cmdDecisionsList.Flags().SortFlags = false
+	cmdDecisionsList.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
+	cmdDecisionsList.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
+	cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
+	cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
+	cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
+	cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
+	cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
+	cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
+	cmdDecisionsList.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
+	cmdDecisionsList.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
+	cmdDecisions.AddCommand(cmdDecisionsList)
+
+	var (
+		addIP       string
+		addRange    string
+		addDuration string
+		addValue    string
+		addScope    string
+		addReason   string
+		addType     string
+	)
+
+	var cmdDecisionsAdd = &cobra.Command{
+		Use:   "add [options]",
+		Short: "Add decision to LAPI",
+		Example: `cscli decisions add --ip 1.2.3.4
+cscli decisions add --range 1.2.3.0/24
+cscli decisions add --ip 1.2.3.4 --duration 24h --type captcha
+cscli decisions add --scope username --value foobar
+`,
+		/*TBD : fix long and example*/
+		Args: cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			var startIP, endIP int64
+			var err error
+			var ip, ipRange string
+			alerts := models.AddAlertsRequest{}
+			origin := "cscli"
+			capacity := int32(0)
+			leakSpeed := "0"
+			eventsCount := int32(1)
+			empty := ""
+			simulated := false
+			startAt := time.Now().Format(time.RFC3339)
+			stopAt := time.Now().Format(time.RFC3339)
+			createdAt := time.Now().Format(time.RFC3339)
+
+			/*take care of shorthand options*/
+			if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
+				log.Fatalf("%s", err)
+			}
+
+			if addIP != "" {
+				addValue = addIP
+				addScope = types.Ip
+			} else if addRange != "" {
+				addValue = addRange
+				addScope = types.Range
+			} else if addValue == "" {
+				cmd.Help()
+				log.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
+				return
+			}
+
+			if addScope == types.Ip {
+				startIP, endIP, err = database.GetIpsFromIpRange(addValue + "/32")
+				if err != nil {
+					log.Fatalf("unable to parse IP : '%s'", addValue)
+				}
+			}
+			if addScope == types.Range {
+				startIP, endIP, err = database.GetIpsFromIpRange(addValue)
+				if err != nil {
+					log.Fatalf("unable to parse Range : '%s'", addValue)
+				}
+				ipRange = addValue
+			}
+
+			if addReason == "" {
+				addReason = fmt.Sprintf("manual '%s' from '%s'", addType, csConfig.API.Client.Credentials.Login)
+			}
+
+			decision := models.Decision{
+				Duration: &addDuration,
+				Scope:    &addScope,
+				Value:    &addValue,
+				Type:     &addType,
+				Scenario: &addReason,
+				Origin:   &origin,
+				StartIP:  startIP,
+				EndIP:    endIP,
+			}
+			alert := models.Alert{
+				Capacity:        &capacity,
+				Decisions:       []*models.Decision{&decision},
+				Events:          []*models.Event{},
+				EventsCount:     &eventsCount,
+				Leakspeed:       &leakSpeed,
+				Message:         &addReason,
+				ScenarioHash:    &empty,
+				Scenario:        &addReason,
+				ScenarioVersion: &empty,
+				Simulated:       &simulated,
+				Source: &models.Source{
+					AsName:   empty,
+					AsNumber: empty,
+					Cn:       empty,
+					IP:       ip,
+					Range:    ipRange,
+					Scope:    &addScope,
+					Value:    &addValue,
+				},
+				StartAt:   &startAt,
+				StopAt:    &stopAt,
+				CreatedAt: createdAt,
+			}
+			alerts = append(alerts, &alert)
+
+			_, _, err = Client.Alerts.Add(context.Background(), alerts)
+			if err != nil {
+				log.Fatalf(err.Error())
+			}
+
+			log.Info("Decision successfully added")
+		},
+	}
+
+	cmdDecisionsAdd.Flags().SortFlags = false
+	cmdDecisionsAdd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmdDecisionsAdd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmdDecisionsAdd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
+	cmdDecisionsAdd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
+	cmdDecisionsAdd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
+	cmdDecisionsAdd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
+	cmdDecisionsAdd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
+	cmdDecisions.AddCommand(cmdDecisionsAdd)
+
+	var delFilter = apiclient.DecisionsDeleteOpts{
+		ScopeEquals: new(string),
+		ValueEquals: new(string),
+		TypeEquals:  new(string),
+		IPEquals:    new(string),
+		RangeEquals: new(string),
+	}
+	var delDecisionId string
+	var delDecisionAll bool
+	var cmdDecisionsDelete = &cobra.Command{
+		Use:   "delete [options]",
+		Short: "Delete decisions",
+		Example: `cscli decisions delete -r 1.2.3.0/24
+cscli decisions delete -i 1.2.3.4
+cscli decisions delete -s crowdsecurity/ssh-bf
+cscli decisions delete --id 42
+cscli decisions delete --type captcha
+`,
+		/*TBD : refaire le Long/Example*/
+		PreRun: func(cmd *cobra.Command, args []string) {
+			if delDecisionAll {
+				return
+			}
+			if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
+				*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
+				*delFilter.RangeEquals == "" && delDecisionId == "" {
+				cmd.Usage()
+				log.Fatalln("At least one filter or --all must be specified")
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			var decisions *models.DeleteDecisionResponse
+
+			/*take care of shorthand options*/
+			if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
+				log.Fatalf("%s", err)
+			}
+			if *delFilter.ScopeEquals == "" {
+				delFilter.ScopeEquals = nil
+			}
+			if *delFilter.ValueEquals == "" {
+				delFilter.ValueEquals = nil
+			}
+
+			if *delFilter.TypeEquals == "" {
+				delFilter.TypeEquals = nil
+			}
+
+			if *delFilter.IPEquals == "" {
+				delFilter.IPEquals = nil
+			}
+
+			if *delFilter.RangeEquals == "" {
+				delFilter.RangeEquals = nil
+			}
+
+			if delDecisionId == "" {
+				decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
+				if err != nil {
+					log.Fatalf("Unable to delete decisions : %v", err.Error())
+				}
+			} else {
+				decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
+				if err != nil {
+					log.Fatalf("Unable to delete decision : %v", err.Error())
+				}
+			}
+			log.Infof("%s decision(s) deleted", decisions.NbDeleted)
+		},
+	}
+
+	cmdDecisionsDelete.Flags().SortFlags = false
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmdDecisionsDelete.Flags().StringVar(&delDecisionId, "id", "", "decision id")
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmdDecisionsDelete.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
+	cmdDecisions.AddCommand(cmdDecisionsDelete)
+
+	return cmdDecisions
+}

cmd/crowdsec-cli/hub.go

@@ -0,0 +1,87 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func NewHubCmd() *cobra.Command {
+	/* ---- HUB COMMAND */
+	var cmdHub = &cobra.Command{
+		Use:   "hub [action]",
+		Short: "Manage Hub",
+		Long: `
+Hub management
+
+List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
+Hub is manage by cscli, to get latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
+		`,
+		Example: `
+cscli hub list   # List all installed configurations
+cscli hub update # Download list of available configurations from the hub
+		`,
+		Args: cobra.ExactArgs(0),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			return nil
+		},
+	}
+	cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
+
+	var cmdHubList = &cobra.Command{
+		Use:   "list [-a]",
+		Short: "List installed configs",
+		Args:  cobra.ExactArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			cwhub.DisplaySummary()
+			log.Printf("PARSERS:")
+			ListItem(cwhub.PARSERS, args)
+			log.Printf("SCENARIOS:")
+			ListItem(cwhub.SCENARIOS, args)
+			log.Printf("COLLECTIONS:")
+			ListItem(cwhub.COLLECTIONS, args)
+			log.Printf("POSTOVERFLOWS:")
+			ListItem(cwhub.PARSERS_OVFLW, args)
+		},
+	}
+	cmdHub.PersistentFlags().BoolVarP(&listAll, "all", "a", false, "List as well disabled items")
+	cmdHub.AddCommand(cmdHubList)
+
+	var cmdHubUpdate = &cobra.Command{
+		Use:   "update",
+		Short: "Fetch available configs from hub",
+		Long: `
+Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
+`,
+		Args: cobra.ExactArgs(0),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := setHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.UpdateHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+		},
+	}
+	cmdHub.AddCommand(cmdHubUpdate)
+
+	return cmdHub
+}

cmd/crowdsec-cli/inspect.go

@@ -1,110 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"gopkg.in/yaml.v2"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/spf13/cobra"
-)
-
-func InspectItem(name string, objectType string) {
-
-	for _, hubItem := range cwhub.HubIdx[objectType] {
-		if hubItem.Name != name {
-			continue
-		}
-		buff, err := yaml.Marshal(hubItem)
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-		fmt.Printf("%s", string(buff))
-	}
-}
-
-func NewInspectCmd() *cobra.Command {
-	var cmdInspect = &cobra.Command{
-		Use:   "inspect [type] [config]",
-		Short: "Inspect configuration(s)",
-		Long: `
-Inspect give you full detail about local installed configuration.
-
-[type] must be parser, scenario, postoverflow, collection.
-
-[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed.
-`,
-		Example: `cscli inspect parser crowdsec/xxx  
-cscli inspect collection crowdsec/xxx`,
-		Args: cobra.MinimumNArgs(1),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-			return nil
-		},
-	}
-
-	var cmdInspectParser = &cobra.Command{
-		Use:     "parser [config]",
-		Short:   "Inspect given log parser",
-		Long:    `Inspect given parser from hub`,
-		Example: `cscli inspect parser crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			InspectItem(args[0], cwhub.PARSERS)
-		},
-	}
-	cmdInspect.AddCommand(cmdInspectParser)
-	var cmdInspectScenario = &cobra.Command{
-		Use:     "scenario [config]",
-		Short:   "Inspect given scenario",
-		Long:    `Inspect given scenario from hub`,
-		Example: `cscli inspect scenario crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			InspectItem(args[0], cwhub.SCENARIOS)
-		},
-	}
-	cmdInspect.AddCommand(cmdInspectScenario)
-
-	var cmdInspectCollection = &cobra.Command{
-		Use:     "collection [config]",
-		Short:   "Inspect given collection",
-		Long:    `Inspect given collection from hub`,
-		Example: `cscli inspect collection crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			InspectItem(args[0], cwhub.COLLECTIONS)
-		},
-	}
-	cmdInspect.AddCommand(cmdInspectCollection)
-
-	var cmdInspectPostoverflow = &cobra.Command{
-		Use:     "postoverflow [config]",
-		Short:   "Inspect given postoverflow parser",
-		Long:    `Inspect given postoverflow from hub.`,
-		Example: `cscli inspect postoverflow crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			InspectItem(args[0], cwhub.PARSERS_OVFLW)
-		},
-	}
-	cmdInspect.AddCommand(cmdInspectPostoverflow)
-
-	return cmdInspect
-}

cmd/crowdsec-cli/install.go

@@ -1,148 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/spf13/cobra"
-)
-
-var download_only, force_install bool
-
-func InstallItem(name string, obtype string) {
-	for _, it := range cwhub.HubIdx[obtype] {
-		if it.Name == name {
-			if download_only && it.Downloaded && it.UpToDate {
-				log.Warningf("%s is already downloaded and up-to-date", it.Name)
-				return
-			}
-			it, err := cwhub.DownloadLatest(it, cwhub.Hubdir, force_install, config.DataFolder)
-			if err != nil {
-				log.Fatalf("error while downloading %s : %v", it.Name, err)
-			}
-			cwhub.HubIdx[obtype][it.Name] = it
-			if download_only {
-				log.Infof("Downloaded %s to %s", it.Name, cwhub.Hubdir+"/"+it.RemotePath)
-				return
-			}
-			it, err = cwhub.EnableItem(it, cwhub.Installdir, cwhub.Hubdir)
-			if err != nil {
-				log.Fatalf("error while enabled %s : %v.", it.Name, err)
-			}
-			cwhub.HubIdx[obtype][it.Name] = it
-			log.Infof("Enabled %s", it.Name)
-			return
-		}
-	}
-	log.Warningf("%s not found in hub index", name)
-	/*iterate of pkg index data*/
-}
-
-func NewInstallCmd() *cobra.Command {
-	/* ---- INSTALL COMMAND */
-
-	var cmdInstall = &cobra.Command{
-		Use:   "install [type] [config]",
-		Short: "Install configuration(s) from hub",
-		Long: `
-Install configuration from the CrowdSec Hub.
-
-In order to download latest versions of configuration, 
-you should [update cscli](./cscli_update.md).
-
-[type] must be parser, scenario, postoverflow, collection.
-
-[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net).
-`,
-		Example: `cscli install [type] [config_name]`,
-		Args:    cobra.MinimumNArgs(1),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := setHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
-		},
-	}
-	cmdInstall.PersistentFlags().BoolVarP(&download_only, "download-only", "d", false, "Only download packages, don't enable")
-	cmdInstall.PersistentFlags().BoolVar(&force_install, "force", false, "Force install : Overwrite tainted and outdated files")
-
-	var cmdInstallParser = &cobra.Command{
-		Use:     "parser [config]",
-		Short:   "Install given parser",
-		Long:    `Fetch and install given parser from hub`,
-		Example: `cscli install parser crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			for _, name := range args {
-				InstallItem(name, cwhub.PARSERS)
-			}
-		},
-	}
-	cmdInstall.AddCommand(cmdInstallParser)
-	var cmdInstallScenario = &cobra.Command{
-		Use:     "scenario [config]",
-		Short:   "Install given scenario",
-		Long:    `Fetch and install given scenario from hub`,
-		Example: `cscli install scenario crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			for _, name := range args {
-				InstallItem(name, cwhub.SCENARIOS)
-			}
-		},
-	}
-	cmdInstall.AddCommand(cmdInstallScenario)
-
-	var cmdInstallCollection = &cobra.Command{
-		Use:     "collection [config]",
-		Short:   "Install given collection",
-		Long:    `Fetch and install given collection from hub`,
-		Example: `cscli install collection crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			for _, name := range args {
-				InstallItem(name, cwhub.COLLECTIONS)
-			}
-		},
-	}
-	cmdInstall.AddCommand(cmdInstallCollection)
-
-	var cmdInstallPostoverflow = &cobra.Command{
-		Use:   "postoverflow [config]",
-		Short: "Install given postoverflow parser",
-		Long: `Fetch and install given postoverflow from hub.
-As a reminder, postoverflows are parsing configuration that will occur after the overflow (before a decision is applied).`,
-		Example: `cscli install collection crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("failed to get Hub index : %v", err)
-			}
-			for _, name := range args {
-				InstallItem(name, cwhub.PARSERS_OVFLW)
-			}
-		},
-	}
-	cmdInstall.AddCommand(cmdInstallPostoverflow)
-
-	return cmdInstall
-}

cmd/crowdsec-cli/lapi.go

@@ -0,0 +1,167 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http/httputil"
+	"net/url"
+	"strings"
+
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/go-openapi/strfmt"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+var LAPIURLPrefix string = "v1"
+
+func NewLapiCmd() *cobra.Command {
+	var cmdLapi = &cobra.Command{
+		Use:   "lapi [action]",
+		Short: "Manage interaction with Local API (LAPI)",
+		Args:  cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.API.Client == nil {
+				log.Fatalln("There is no API->client configuration")
+			}
+			if csConfig.API.Client.Credentials == nil {
+				log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.Self)
+			}
+			return nil
+		},
+	}
+
+	var cmdLapiRegister = &cobra.Command{
+		Use:   "register",
+		Short: "Register a machine to Local API (LAPI)",
+		Long: `Register you machine to the Local API (LAPI).
+Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective.`,
+		Args: cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			id, err := generateID()
+			if err != nil {
+				log.Fatalf("unable to generate machine id: %s", err)
+			}
+			password := strfmt.Password(generatePassword(passwordLength))
+			if apiURL == "" {
+				if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
+					apiURL = csConfig.API.Client.Credentials.URL
+				} else {
+					log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
+				}
+			}
+			/*URL needs to end with /, but user doesn't care*/
+			if !strings.HasSuffix(apiURL, "/") {
+				apiURL += "/"
+			}
+			/*URL needs to start with http://, but user doesn't care*/
+			if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") {
+				apiURL = "http://" + apiURL
+			}
+			apiurl, err := url.Parse(apiURL)
+			if err != nil {
+				log.Fatalf("parsing api url: %s", err)
+			}
+			_, err = apiclient.RegisterClient(&apiclient.Config{
+				MachineID:     id,
+				Password:      password,
+				UserAgent:     fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
+				URL:           apiurl,
+				VersionPrefix: LAPIURLPrefix,
+			}, nil)
+
+			if err != nil {
+				log.Fatalf("api client register: %s", err)
+			}
+
+			var dumpFile string
+			if outputFile != "" {
+				dumpFile = outputFile
+			} else if csConfig.API.Client.CredentialsFilePath != "" {
+				dumpFile = csConfig.API.Client.CredentialsFilePath
+			} else {
+				dumpFile = ""
+			}
+			apiCfg := csconfig.ApiCredentialsCfg{
+				Login:    id,
+				Password: password.String(),
+				URL:      apiURL,
+			}
+			apiConfigDump, err := yaml.Marshal(apiCfg)
+			if err != nil {
+				log.Fatalf("unable to marshal api credentials: %s", err)
+			}
+			if dumpFile != "" {
+				err = ioutil.WriteFile(dumpFile, apiConfigDump, 0644)
+				if err != nil {
+					log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+				}
+				log.Printf("API credentials dumped to '%s'", dumpFile)
+			} else {
+				fmt.Printf("%s\n", string(apiConfigDump))
+			}
+			log.Warningf("Run 'systemctl reload crowdsec' for the new configuration to be effective")
+		},
+	}
+	cmdLapiRegister.Flags().StringVarP(&apiURL, "url", "u", "", "URL of the API (ie. http://127.0.0.1)")
+	cmdLapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
+	cmdLapi.AddCommand(cmdLapiRegister)
+
+	var cmdLapiStatus = &cobra.Command{
+		Use:   "status",
+		Short: "Check authentication to Local API (LAPI)",
+		Args:  cobra.MinimumNArgs(0),
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
+			password := strfmt.Password(csConfig.API.Client.Credentials.Password)
+			apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
+			login := csConfig.API.Client.Credentials.Login
+			if err != nil {
+				log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
+			}
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to load hub index : %s", err)
+			}
+			scenarios, err := cwhub.GetUpstreamInstalledScenariosAsString()
+			if err != nil {
+				log.Fatalf("failed to get scenarios : %s", err.Error())
+			}
+
+			Client, err = apiclient.NewDefaultClient(apiurl,
+				LAPIURLPrefix,
+				fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
+				nil)
+			if err != nil {
+				log.Fatalf("init default client: %s", err)
+			}
+			t := models.WatcherAuthRequest{
+				MachineID: &login,
+				Password:  &password,
+				Scenarios: scenarios,
+			}
+			log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
+			log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
+			resp, err := Client.Auth.AuthenticateWatcher(context.Background(), t)
+			if err != nil {
+				log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
+			} else {
+				log.Infof("You can successfully interact with Local API (LAPI)")
+			}
+			for k, v := range resp.Response.Header {
+				log.Debugf("[headers] %s : %s", k, v)
+			}
+			dump, _ := httputil.DumpResponse(resp.Response, true)
+			log.Debugf("Response: %s", string(dump))
+		},
+	}
+	cmdLapi.AddCommand(cmdLapiStatus)
+	return cmdLapi
+}

cmd/crowdsec-cli/list.go

@@ -1,153 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-
-	"github.com/enescakir/emoji"
-	"github.com/olekukonko/tablewriter"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-var listAll bool
-
-func doListing(ttype string, args []string) {
-
-	var pkgst []map[string]string
-
-	if len(args) == 1 {
-		pkgst = cwhub.HubStatus(ttype, args[0], listAll)
-	} else {
-		pkgst = cwhub.HubStatus(ttype, "", listAll)
-	}
-
-	if config.output == "human" {
-
-		table := tablewriter.NewWriter(os.Stdout)
-		table.SetCenterSeparator("")
-		table.SetColumnSeparator("")
-
-		table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
-		table.SetAlignment(tablewriter.ALIGN_LEFT)
-		table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
-		for _, v := range pkgst {
-			table.Append([]string{v["name"], v["utf8_status"], v["local_version"], v["local_path"]})
-		}
-		table.Render()
-	} else if config.output == "json" {
-		x, err := json.MarshalIndent(pkgst, "", " ")
-		if err != nil {
-			log.Fatalf("failed to unmarshal")
-		}
-		fmt.Printf("%s", string(x))
-	} else if config.output == "raw" {
-		for _, v := range pkgst {
-			fmt.Printf("%s %s\n", v["name"], v["description"])
-		}
-	}
-}
-
-func NewListCmd() *cobra.Command {
-	/* ---- LIST COMMAND */
-	var cmdList = &cobra.Command{
-		Use:   "list [-a]",
-		Short: "List enabled configs",
-		Long: `
-List enabled configurations (parser/scenarios/collections) on your host.
-
-It is possible to list also configuration from [Crowdsec Hub](https://hub.crowdsec.net) with the '-a' options.
-
-[type] must be parsers, scenarios, postoverflows, collections
-		`,
-		Example: `cscli list  # List all local configurations
-cscli list [type] # List all local configuration of type [type]
-cscli list -a # List all local and remote configurations
-		`,
-		Args: cobra.ExactArgs(0),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			return nil
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-
-			cwhub.DisplaySummary()
-			log.Printf("PARSERS:")
-			doListing(cwhub.PARSERS, args)
-			log.Printf("SCENARIOS:")
-			doListing(cwhub.SCENARIOS, args)
-			log.Printf("COLLECTIONS:")
-			doListing(cwhub.COLLECTIONS, args)
-			log.Printf("POSTOVERFLOWS:")
-			doListing(cwhub.PARSERS_OVFLW, args)
-		},
-	}
-	cmdList.PersistentFlags().BoolVarP(&listAll, "all", "a", false, "List as well disabled items")
-
-	var cmdListParsers = &cobra.Command{
-		Use:   "parsers [-a]",
-		Short: "List enabled parsers",
-		Long:  ``,
-		Args:  cobra.ExactArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			doListing(cwhub.PARSERS, args)
-		},
-	}
-	cmdList.AddCommand(cmdListParsers)
-
-	var cmdListScenarios = &cobra.Command{
-		Use:   "scenarios [-a]",
-		Short: "List enabled scenarios",
-		Long:  ``,
-		Args:  cobra.ExactArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			doListing(cwhub.SCENARIOS, args)
-		},
-	}
-	cmdList.AddCommand(cmdListScenarios)
-
-	var cmdListCollections = &cobra.Command{
-		Use:   "collections [-a]",
-		Short: "List enabled collections",
-		Long:  ``,
-		Args:  cobra.ExactArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			doListing(cwhub.COLLECTIONS, args)
-		},
-	}
-	cmdList.AddCommand(cmdListCollections)
-
-	var cmdListPostoverflows = &cobra.Command{
-		Use:   "postoverflows [-a]",
-		Short: "List enabled postoverflow parsers",
-		Long:  ``,
-		Args:  cobra.ExactArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			doListing(cwhub.PARSERS_OVFLW, args)
-		},
-	}
-	cmdList.AddCommand(cmdListPostoverflows)
-
-	return cmdList
-}

cmd/crowdsec-cli/machines.go

@@ -0,0 +1,305 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/database"
+	"github.com/denisbrodbeck/machineid"
+	"github.com/enescakir/emoji"
+	"github.com/go-openapi/strfmt"
+	"github.com/olekukonko/tablewriter"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
+)
+
+var machineID string
+var machinePassword string
+var interactive bool
+var apiURL string
+var outputFile string
+var forceAdd bool
+var autoAdd bool
+
+var (
+	passwordLength = 64
+	upper          = "ABCDEFGHIJKLMNOPQRSTUVWXY"
+	lower          = "abcdefghijklmnopqrstuvwxyz"
+	digits         = "0123456789"
+)
+
+const (
+	uuid = "/proc/sys/kernel/random/uuid"
+)
+
+func generatePassword(length int) string {
+	rand.Seed(time.Now().UnixNano())
+	charset := upper + lower + digits
+
+	buf := make([]byte, length)
+	buf[0] = digits[rand.Intn(len(digits))]
+	buf[1] = upper[rand.Intn(len(upper))]
+	buf[2] = lower[rand.Intn(len(lower))]
+
+	for i := 3; i < length; i++ {
+		buf[i] = charset[rand.Intn(len(charset))]
+	}
+	rand.Shuffle(len(buf), func(i, j int) {
+		buf[i], buf[j] = buf[j], buf[i]
+	})
+
+	return string(buf)
+}
+
+func generateID() (string, error) {
+	id, err := machineid.ID()
+	if err != nil {
+		log.Debugf("failed to get machine-id with usual files : %s", err)
+	}
+	if id == "" || err != nil {
+		bID, err := ioutil.ReadFile(uuid)
+		if err != nil {
+			return "", errors.Wrap(err, "generating machine id")
+		}
+		id = string(bID)
+		id = strings.ReplaceAll(id, "-", "")[:32]
+	}
+	id = fmt.Sprintf("%s%s", id, generatePassword(16))
+	return id, nil
+}
+
+func NewMachinesCmd() *cobra.Command {
+	/* ---- DECISIONS COMMAND */
+	var cmdMachines = &cobra.Command{
+		Use:   "machines [action]",
+		Short: "Manage local API machines",
+		Long: `
+Machines Management.
+
+To list/add/delete/register/validate machines
+`,
+		Example: `cscli machines [action]`,
+	}
+
+	var cmdMachinesList = &cobra.Command{
+		Use:     "list",
+		Short:   "List machines",
+		Long:    `List `,
+		Example: `cscli machines list`,
+		Args:    cobra.MaximumNArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				log.Fatalf("unable to create new database client: %s", err)
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			machines, err := dbClient.ListMachines()
+			if err != nil {
+				log.Errorf("unable to list blockers: %s", err)
+			}
+			if csConfig.Cscli.Output == "human" {
+				table := tablewriter.NewWriter(os.Stdout)
+				table.SetCenterSeparator("")
+				table.SetColumnSeparator("")
+
+				table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+				table.SetAlignment(tablewriter.ALIGN_LEFT)
+				table.SetHeader([]string{"Name", "IP Address", "Last Update", "Status", "Version"})
+				for _, w := range machines {
+					var validated string
+					if w.IsValidated {
+						validated = fmt.Sprintf("%s", emoji.CheckMark)
+					} else {
+						validated = fmt.Sprintf("%s", emoji.Prohibited)
+					}
+					table.Append([]string{w.MachineId, w.IpAddress, w.UpdatedAt.Format(time.RFC3339), validated, w.Version})
+				}
+				table.Render()
+			} else if csConfig.Cscli.Output == "json" {
+				x, err := json.MarshalIndent(machines, "", " ")
+				if err != nil {
+					log.Fatalf("failed to unmarshal")
+				}
+				fmt.Printf("%s", string(x))
+			} else if csConfig.Cscli.Output == "raw" {
+				for _, w := range machines {
+					var validated string
+					if w.IsValidated {
+						validated = "true"
+					} else {
+						validated = "false"
+					}
+					fmt.Printf("%s,%s,%s,%s,%s\n", w.MachineId, w.IpAddress, w.UpdatedAt.Format(time.RFC3339), validated, w.Version)
+				}
+			} else {
+				log.Errorf("unknown output '%s'", csConfig.Cscli.Output)
+			}
+		},
+	}
+	cmdMachines.AddCommand(cmdMachinesList)
+
+	var cmdMachinesAdd = &cobra.Command{
+		Use:   "add",
+		Short: "add machine to the database.",
+		Long:  `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
+		Example: `
+cscli machines add --auto
+cscli machines add MyTestMachine --auto
+cscli machines add MyTestMachine --password MyPassword
+`,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				log.Fatalf("unable to create new database client: %s", err)
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			var dumpFile string
+			var err error
+
+			// create machineID if doesn't specified by user
+			if len(args) == 0 {
+				if !autoAdd {
+					err = cmd.Help()
+					if err != nil {
+						log.Fatalf("unable to print help(): %s", err)
+					}
+					return
+				}
+				machineID, err = generateID()
+				if err != nil {
+					log.Fatalf("unable to generate machine id : %s", err)
+				}
+			} else {
+				machineID = args[0]
+			}
+
+			/*check if file already exists*/
+			if outputFile != "" {
+				dumpFile = outputFile
+			} else if csConfig.API.Client.CredentialsFilePath != "" {
+				dumpFile = csConfig.API.Client.CredentialsFilePath
+			}
+
+			// create password if doesn't specified by user
+			if machinePassword == "" && !interactive {
+				if !autoAdd {
+					err = cmd.Help()
+					if err != nil {
+						log.Fatalf("unable to print help(): %s", err)
+					}
+					return
+				}
+				machinePassword = generatePassword(passwordLength)
+			} else if machinePassword == "" && interactive {
+				qs := &survey.Password{
+					Message: "Please provide a password for the machine",
+				}
+				survey.AskOne(qs, &machinePassword)
+			}
+			password := strfmt.Password(machinePassword)
+			_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd)
+			if err != nil {
+				log.Fatalf("unable to create machine: %s", err)
+			}
+			log.Infof("Machine '%s' created successfully", machineID)
+
+			if apiURL == "" {
+				if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
+					apiURL = csConfig.API.Client.Credentials.URL
+				} else if csConfig.API.Server != nil && csConfig.API.Server.ListenURI != "" {
+					apiURL = "http://" + csConfig.API.Server.ListenURI
+				} else {
+					log.Fatalf("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
+				}
+			}
+			apiCfg := csconfig.ApiCredentialsCfg{
+				Login:    machineID,
+				Password: password.String(),
+				URL:      apiURL,
+			}
+			apiConfigDump, err := yaml.Marshal(apiCfg)
+			if err != nil {
+				log.Fatalf("unable to marshal api credentials: %s", err)
+			}
+			if dumpFile != "" {
+				err = ioutil.WriteFile(dumpFile, apiConfigDump, 0644)
+				if err != nil {
+					log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+				}
+				log.Printf("API credentials dumped to '%s'", dumpFile)
+			} else {
+				fmt.Printf("%s\n", string(apiConfigDump))
+			}
+		},
+	}
+	cmdMachinesAdd.Flags().StringVarP(&machinePassword, "password", "p", "", "machine password to login to the API")
+	cmdMachinesAdd.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
+	cmdMachinesAdd.Flags().StringVarP(&apiURL, "url", "u", "", "URL of the local API")
+	cmdMachinesAdd.Flags().BoolVarP(&interactive, "interactive", "i", false, "interfactive mode to enter the password")
+	cmdMachinesAdd.Flags().BoolVarP(&autoAdd, "auto", "a", false, "add the machine automatically (will generate also the username if not provided)")
+	cmdMachinesAdd.Flags().BoolVar(&forceAdd, "force", false, "will force add the machine if it already exist")
+	cmdMachines.AddCommand(cmdMachinesAdd)
+
+	var cmdMachinesDelete = &cobra.Command{
+		Use:     "delete --machine MyTestMachine",
+		Short:   "delete machines",
+		Example: `cscli machines delete <machine_name>`,
+		Args:    cobra.ExactArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				log.Fatalf("unable to create new database client: %s", err)
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			machineID = args[0]
+			err := dbClient.DeleteWatcher(machineID)
+			if err != nil {
+				log.Errorf("unable to create blocker: %s", err)
+				return
+			}
+			log.Infof("machine '%s' deleted successfully", machineID)
+		},
+	}
+	cmdMachinesDelete.Flags().StringVarP(&machineID, "machine", "m", "", "machine to delete")
+	cmdMachines.AddCommand(cmdMachinesDelete)
+
+	var cmdMachinesValidate = &cobra.Command{
+		Use:     "validate",
+		Short:   "validate a machine to access the local API",
+		Long:    `validate a machine to access the local API.`,
+		Example: `cscli machines validate <machine_name>`,
+		Args:    cobra.ExactArgs(1),
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				log.Fatalf("unable to create new database client: %s", err)
+			}
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			machineID = args[0]
+			if err := dbClient.ValidateMachine(machineID); err != nil {
+				log.Fatalf("unable to validate machine '%s': %s", machineID, err)
+			}
+			log.Infof("machine '%s' validated successfuly", machineID)
+		},
+	}
+	cmdMachines.AddCommand(cmdMachinesValidate)
+
+	return cmdMachines
+}

cmd/crowdsec-cli/main.go

@@ -1,26 +1,40 @@
 package main
 
 import (
-	"os/user"
-	"path/filepath"
-	"strings"
-
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/database"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
 )
 
-var dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
+var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
+
+var ConfigFilePath string
+var csConfig *csconfig.GlobalConfig
+var dbClient *database.Client
+
+var OutputFormat string
 
-var config cliConfig
+var downloadOnly bool
+var forceInstall bool
+var forceUpgrade bool
+var removeAll bool
+var purgeRemove bool
+var upgradeAll bool
+var listAll bool
+var restoreOldBackup bool
+
+var prometheusURL string
 
 func initConfig() {
 
-	if dbg_lvl {
+	if trace_lvl {
+		log.SetLevel(log.TraceLevel)
+	} else if dbg_lvl {
 		log.SetLevel(log.DebugLevel)
 	} else if nfo_lvl {
 		log.SetLevel(log.InfoLevel)
@@ -29,42 +43,33 @@ func initConfig() {
 	} else if err_lvl {
 		log.SetLevel(log.ErrorLevel)
 	}
-	if config.output == "json" {
-		log.SetLevel(log.WarnLevel)
-		log.SetFormatter(&log.JSONFormatter{})
-	} else if config.output == "raw" {
-		log.SetLevel(log.ErrorLevel)
-	}
 
-	csConfig := csconfig.NewCrowdSecConfig()
-	if err := csConfig.LoadConfigurationFile(&config.ConfigFilePath); err != nil {
+	csConfig = csconfig.NewConfig()
+
+	log.Debugf("Using %s as configuration file", ConfigFilePath)
+	if err := csConfig.LoadConfigurationFile(ConfigFilePath); err != nil {
 		log.Fatalf(err.Error())
 	}
-	config.configFolder = filepath.Clean(csConfig.CsCliFolder)
-
-	if strings.HasPrefix(config.configFolder, "~/") {
-		usr, err := user.Current()
-		if err != nil {
-			log.Fatalf("failed to resolve path ~/ : %s", err)
+	if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
+		cwhub.HubBranch = csConfig.Cscli.HubBranch
+	}
+	if OutputFormat != "" {
+		csConfig.Cscli.Output = OutputFormat
+		if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
+			log.Fatalf("output format %s unknown", OutputFormat)
 		}
-		config.configFolder = usr.HomeDir + "/" + config.configFolder[2:]
+	}
+	if csConfig.Cscli.Output == "" {
+		csConfig.Cscli.Output = "human"
 	}
 
-	/*read config*/
-	config.InstallFolder = filepath.Clean(csConfig.ConfigFolder)
-	config.HubFolder = filepath.Clean(config.configFolder + "/hub/")
-	if csConfig.OutputConfig == nil {
-		log.Fatalf("Missing backend plugin configuration in %s", config.ConfigFilePath)
+	if csConfig.Cscli.Output == "json" {
+		log.SetLevel(log.WarnLevel)
+		log.SetFormatter(&log.JSONFormatter{})
+	} else if csConfig.Cscli.Output == "raw" {
+		log.SetLevel(log.ErrorLevel)
 	}
-	config.BackendPluginFolder = filepath.Clean(csConfig.OutputConfig.BackendFolder)
-	config.DataFolder = filepath.Clean(csConfig.DataFolder)
-	//
-	cwhub.Installdir = config.InstallFolder
-	cwhub.Cfgdir = config.configFolder
-	cwhub.Hubdir = config.HubFolder
-	config.configured = true
-	config.SimulationCfg = csConfig.SimulationCfg
-	config.SimulationCfgPath = csConfig.SimulationCfgPath
+
 }
 
 func main() {
@@ -74,22 +79,8 @@ func main() {
 		Short: "cscli allows you to manage crowdsec",
 		Long: `cscli is the main command to interact with your crowdsec service, scenarios & db.
 It is meant to allow you to manage bans, parsers/scenarios/etc, api and generally manage you crowdsec setup.`,
-		Example: `View/Add/Remove bans:  
- - cscli ban list  
- - cscli ban add ip 1.2.3.4 24h 'go away'  
- - cscli ban del 1.2.3.4  
-		
-View/Add/Upgrade/Remove scenarios and parsers:  
- - cscli list  
- - cscli install collection crowdsec/linux-web  
- - cscli remove scenario crowdsec/ssh_enum  
- - cscli upgrade --all  
-
-API interaction:
- - cscli api pull
- - cscli api register
- `}
-	/*TODO : add a remediation type*/
+		/*TBD examples*/
+	}
 	var cmdDocGen = &cobra.Command{
 		Use:    "doc",
 		Short:  "Generate the documentation in `./doc/`. Directory must exist.",
@@ -97,7 +88,7 @@ API interaction:
 		Hidden: true,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := doc.GenMarkdownTree(rootCmd, "./doc/"); err != nil {
-				log.Fatalf("Failed to generate cobra doc")
+				log.Fatalf("Failed to generate cobra doc: %s", err.Error())
 			}
 		},
 	}
@@ -114,16 +105,15 @@ API interaction:
 	}
 	rootCmd.AddCommand(cmdVersion)
 
-	//rootCmd.PersistentFlags().BoolVarP(&config.simulation, "simulate", "s", false, "No action; perform a simulation of events that would occur based on the current arguments.")
-	rootCmd.PersistentFlags().StringVarP(&config.ConfigFilePath, "config", "c", "/etc/crowdsec/config/default.yaml", "path to crowdsec config file")
-
-	rootCmd.PersistentFlags().StringVarP(&config.output, "output", "o", "human", "Output format : human, json, raw.")
+	rootCmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", "/etc/crowdsec/config.yaml", "path to crowdsec config file")
+	rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format : human, json, raw.")
 	rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug.")
 	rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")
 	rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.")
 	rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.")
+	rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace.")
 
-	rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "master", "Override hub branch on github")
+	rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
 	if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
 		log.Fatalf("failed to make branch hidden : %s", err)
 	}
@@ -132,19 +122,22 @@ API interaction:
 	rootCmd.Flags().SortFlags = false
 	rootCmd.PersistentFlags().SortFlags = false
 
-	rootCmd.AddCommand(NewBanCmds())
 	rootCmd.AddCommand(NewConfigCmd())
-	rootCmd.AddCommand(NewInstallCmd())
-	rootCmd.AddCommand(NewListCmd())
-	rootCmd.AddCommand(NewRemoveCmd())
-	rootCmd.AddCommand(NewUpdateCmd())
-	rootCmd.AddCommand(NewUpgradeCmd())
-	rootCmd.AddCommand(NewAPICmd())
+	rootCmd.AddCommand(NewHubCmd())
 	rootCmd.AddCommand(NewMetricsCmd())
-	rootCmd.AddCommand(NewBackupCmd())
 	rootCmd.AddCommand(NewDashboardCmd())
-	rootCmd.AddCommand(NewInspectCmd())
+	rootCmd.AddCommand(NewDecisionsCmd())
+	rootCmd.AddCommand(NewAlertsCmd())
+	//	rootCmd.AddCommand(NewInspectCmd())
 	rootCmd.AddCommand(NewSimulationCmds())
+	rootCmd.AddCommand(NewBouncersCmd())
+	rootCmd.AddCommand(NewMachinesCmd())
+	rootCmd.AddCommand(NewParsersCmd())
+	rootCmd.AddCommand(NewScenariosCmd())
+	rootCmd.AddCommand(NewCollectionsCmd())
+	rootCmd.AddCommand(NewPostOverflowsCmd())
+	rootCmd.AddCommand(NewCapiCmd())
+	rootCmd.AddCommand(NewLapiCmd())
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatalf("While executing root command : %s", err)
 	}

cmd/crowdsec-cli/metrics.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
@@ -19,6 +20,37 @@ import (
 	"github.com/spf13/cobra"
 )
 
+func lapiMetricsToTable(table *tablewriter.Table, stats map[string]map[string]map[string]int) error {
+
+	//stats : machine -> route -> method -> count
+	/*we want consistant display order*/
+	machineKeys := []string{}
+	for k := range stats {
+		machineKeys = append(machineKeys, k)
+	}
+	sort.Strings(machineKeys)
+
+	for _, machine := range machineKeys {
+		//oneRow : route -> method -> count
+		machineRow := stats[machine]
+		for routeName, route := range machineRow {
+			for methodName, count := range route {
+				row := []string{}
+				row = append(row, machine)
+				row = append(row, routeName)
+				row = append(row, methodName)
+				if count != 0 {
+					row = append(row, fmt.Sprintf("%d", count))
+				} else {
+					row = append(row, "-")
+				}
+				table.Append(row)
+			}
+		}
+	}
+	return nil
+}
+
 func metricsToTable(table *tablewriter.Table, stats map[string]map[string]int, keys []string) error {
 
 	var sortedKeys []string
@@ -65,6 +97,7 @@ func ShowPrometheus(url string) {
 	transport.ResponseHeaderTimeout = time.Minute
 
 	go func() {
+		defer types.CatchPanic("crowdsec/ShowPrometheus")
 		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
 		if err != nil {
 			log.Fatalf("failed to fetch prometheus metrics : %v", err)
@@ -77,14 +110,22 @@ func ShowPrometheus(url string) {
 	}
 	log.Debugf("Finished reading prometheus output, %d entries", len(result))
 	/*walk*/
+	lapi_decisions_stats := map[string]struct {
+		NonEmpty int
+		Empty    int
+	}{}
 	acquis_stats := map[string]map[string]int{}
 	parsers_stats := map[string]map[string]int{}
 	buckets_stats := map[string]map[string]int{}
+	lapi_stats := map[string]map[string]int{}
+	lapi_machine_stats := map[string]map[string]map[string]int{}
+	lapi_bouncer_stats := map[string]map[string]map[string]int{}
+
 	for idx, fam := range result {
 		if !strings.HasPrefix(fam.Name, "cs_") {
 			continue
 		}
-		log.Debugf("round %d", idx)
+		log.Tracef("round %d", idx)
 		for _, m := range fam.Metrics {
 			metric := m.(prom2json.Metric)
 			name, ok := metric.Labels["name"]
@@ -96,6 +137,12 @@ func ShowPrometheus(url string) {
 				log.Debugf("no source in Metric %v", metric.Labels)
 			}
 			value := m.(prom2json.Metric).Value
+			machine := metric.Labels["machine"]
+			bouncer := metric.Labels["bouncer"]
+
+			route := metric.Labels["route"]
+			method := metric.Labels["method"]
+
 			fval, err := strconv.ParseFloat(value, 32)
 			if err != nil {
 				log.Errorf("Unexpected int value %s : %s", value, err)
@@ -163,13 +210,48 @@ func ShowPrometheus(url string) {
 					parsers_stats[name] = make(map[string]int)
 				}
 				parsers_stats[name]["unparsed"] += ival
+			case "cs_lapi_route_requests_total":
+				if _, ok := lapi_stats[route]; !ok {
+					lapi_stats[route] = make(map[string]int)
+				}
+				lapi_stats[route][method] += ival
+			case "cs_lapi_machine_requests_total":
+				if _, ok := lapi_machine_stats[machine]; !ok {
+					lapi_machine_stats[machine] = make(map[string]map[string]int)
+				}
+				if _, ok := lapi_machine_stats[machine][route]; !ok {
+					lapi_machine_stats[machine][route] = make(map[string]int)
+				}
+				lapi_machine_stats[machine][route][method] += ival
+			case "cs_lapi_bouncer_requests_total":
+				if _, ok := lapi_bouncer_stats[bouncer]; !ok {
+					lapi_bouncer_stats[bouncer] = make(map[string]map[string]int)
+				}
+				if _, ok := lapi_bouncer_stats[bouncer][route]; !ok {
+					lapi_bouncer_stats[bouncer][route] = make(map[string]int)
+				}
+				lapi_bouncer_stats[bouncer][route][method] += ival
+			case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
+				if _, ok := lapi_decisions_stats[bouncer]; !ok {
+					lapi_decisions_stats[bouncer] = struct {
+						NonEmpty int
+						Empty    int
+					}{}
+				}
+				x := lapi_decisions_stats[bouncer]
+				if fam.Name == "cs_lapi_decisions_ko_total" {
+					x.Empty += ival
+				} else if fam.Name == "cs_lapi_decisions_ok_total" {
+					x.NonEmpty += ival
+				}
+				lapi_decisions_stats[bouncer] = x
 			default:
 				continue
 			}
 
 		}
 	}
-	if config.output == "human" {
+	if csConfig.Cscli.Output == "human" {
 
 		acquisTable := tablewriter.NewWriter(os.Stdout)
 		acquisTable.SetHeader([]string{"Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket"})
@@ -191,22 +273,93 @@ func ShowPrometheus(url string) {
 			log.Warningf("while collecting acquis stats : %s", err)
 		}
 
-		log.Printf("Buckets Metrics:")
-		bucketsTable.Render()
-		log.Printf("Acquisition Metrics:")
-		acquisTable.Render()
-		log.Printf("Parser Metrics:")
-		parsersTable.Render()
-	} else if config.output == "json" {
-		for _, val := range []map[string]map[string]int{acquis_stats, parsers_stats, buckets_stats} {
+		lapiMachinesTable := tablewriter.NewWriter(os.Stdout)
+		lapiMachinesTable.SetHeader([]string{"Machine", "Route", "Method", "Hits"})
+		if err := lapiMetricsToTable(lapiMachinesTable, lapi_machine_stats); err != nil {
+			log.Warningf("while collecting machine lapi stats : %s", err)
+		}
+
+		//lapiMetricsToTable
+		lapiBouncersTable := tablewriter.NewWriter(os.Stdout)
+		lapiBouncersTable.SetHeader([]string{"Bouncer", "Route", "Method", "Hits"})
+		if err := lapiMetricsToTable(lapiBouncersTable, lapi_bouncer_stats); err != nil {
+			log.Warningf("while collecting bouncer lapi stats : %s", err)
+		}
+
+		lapiDecisionsTable := tablewriter.NewWriter(os.Stdout)
+		lapiDecisionsTable.SetHeader([]string{"Bouncer", "Empty answers", "Non-empty answers"})
+		for bouncer, hits := range lapi_decisions_stats {
+			row := []string{}
+			row = append(row, bouncer)
+			row = append(row, fmt.Sprintf("%d", hits.Empty))
+			row = append(row, fmt.Sprintf("%d", hits.NonEmpty))
+			lapiDecisionsTable.Append(row)
+		}
+
+		/*unfortunately, we can't reuse metricsToTable as the structure is too different :/*/
+		lapiTable := tablewriter.NewWriter(os.Stdout)
+		lapiTable.SetHeader([]string{"Route", "Method", "Hits"})
+		sortedKeys := []string{}
+		for akey := range lapi_stats {
+			sortedKeys = append(sortedKeys, akey)
+		}
+		sort.Strings(sortedKeys)
+		for _, alabel := range sortedKeys {
+			astats := lapi_stats[alabel]
+			subKeys := []string{}
+			for skey := range astats {
+				subKeys = append(subKeys, skey)
+			}
+			sort.Strings(subKeys)
+			for _, sl := range subKeys {
+				row := []string{}
+				row = append(row, alabel)
+				row = append(row, sl)
+				row = append(row, fmt.Sprintf("%d", astats[sl]))
+				lapiTable.Append(row)
+			}
+		}
+
+		if bucketsTable.NumLines() > 0 {
+			log.Printf("Buckets Metrics:")
+			bucketsTable.Render()
+		}
+		if acquisTable.NumLines() > 0 {
+			log.Printf("Acquisition Metrics:")
+			acquisTable.Render()
+		}
+		if parsersTable.NumLines() > 0 {
+			log.Printf("Parser Metrics:")
+			parsersTable.Render()
+		}
+		if lapiTable.NumLines() > 0 {
+			log.Printf("Local Api Metrics:")
+			lapiTable.Render()
+		}
+		if lapiMachinesTable.NumLines() > 0 {
+			log.Printf("Local Api Machines Metrics:")
+			lapiMachinesTable.Render()
+		}
+		if lapiBouncersTable.NumLines() > 0 {
+			log.Printf("Local Api Bouncers Metrics:")
+			lapiBouncersTable.Render()
+		}
+
+		if lapiDecisionsTable.NumLines() > 0 {
+			log.Printf("Local Api Bouncers Decisions:")
+			lapiDecisionsTable.Render()
+		}
+
+	} else if csConfig.Cscli.Output == "json" {
+		for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats} {
 			x, err := json.MarshalIndent(val, "", " ")
 			if err != nil {
 				log.Fatalf("failed to unmarshal metrics : %v", err)
 			}
 			fmt.Printf("%s\n", string(x))
 		}
-	} else if config.output == "raw" {
-		for _, val := range []map[string]map[string]int{acquis_stats, parsers_stats, buckets_stats} {
+	} else if csConfig.Cscli.Output == "raw" {
+		for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats} {
 			x, err := yaml.Marshal(val)
 			if err != nil {
 				log.Fatalf("failed to unmarshal metrics : %v", err)
@@ -216,8 +369,6 @@ func ShowPrometheus(url string) {
 	}
 }
 
-var purl string
-
 func NewMetricsCmd() *cobra.Command {
 	/* ---- UPDATE COMMAND */
 	var cmdMetrics = &cobra.Command{
@@ -226,10 +377,10 @@ func NewMetricsCmd() *cobra.Command {
 		Long:  `Fetch metrics from the prometheus server and display them in a human-friendly way`,
 		Args:  cobra.ExactArgs(0),
 		Run: func(cmd *cobra.Command, args []string) {
-			ShowPrometheus(purl)
+			ShowPrometheus(prometheusURL)
 		},
 	}
-	cmdMetrics.PersistentFlags().StringVarP(&purl, "url", "u", "http://127.0.0.1:6060/metrics", "Prometheus url")
+	cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "http://127.0.0.1:6060/metrics", "Prometheus url")
 
 	return cmdMetrics
 }

cmd/crowdsec-cli/parsers.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+func NewParsersCmd() *cobra.Command {
+	var cmdParsers = &cobra.Command{
+		Use:   "parsers [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
+		Example: `cscli parsers install crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/sshd-logs
+cscli parsers list
+cscli parsers remove crowdsecurity/sshd-logs
+`,
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := setHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
+		},
+	}
+
+	var cmdParsersInstall = &cobra.Command{
+		Use:     "install [config]",
+		Short:   "Install given parser(s)",
+		Long:    `Fetch and install given parser(s) from hub`,
+		Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			for _, name := range args {
+				InstallItem(name, cwhub.PARSERS, forceInstall)
+			}
+		},
+	}
+	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdParsersInstall.PersistentFlags().BoolVar(&forceInstall, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdParsers.AddCommand(cmdParsersInstall)
+
+	var cmdParsersRemove = &cobra.Command{
+		Use:     "remove [config]",
+		Short:   "Remove given parser(s)",
+		Long:    `Remove given parse(s) from hub`,
+		Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			if removeAll {
+				RemoveMany(cwhub.PARSERS, "")
+			} else {
+				for _, name := range args {
+					RemoveMany(cwhub.PARSERS, name)
+				}
+			}
+		},
+	}
+	cmdParsersRemove.PersistentFlags().BoolVar(&purgeRemove, "purge", false, "Delete source file too")
+	cmdParsersRemove.PersistentFlags().BoolVar(&removeAll, "all", false, "Delete all the parsers")
+	cmdParsers.AddCommand(cmdParsersRemove)
+
+	var cmdParsersUpgrade = &cobra.Command{
+		Use:     "upgrade [config]",
+		Short:   "Upgrade given parser(s)",
+		Long:    `Fetch and upgrade given parser(s) from hub`,
+		Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if upgradeAll {
+				UpgradeConfig(cwhub.PARSERS, "", forceUpgrade)
+			} else {
+				for _, name := range args {
+					UpgradeConfig(cwhub.PARSERS, name, forceUpgrade)
+				}
+			}
+		},
+	}
+	cmdParsersUpgrade.PersistentFlags().BoolVar(&upgradeAll, "all", false, "Upgrade all the parsers")
+	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceUpgrade, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdParsers.AddCommand(cmdParsersUpgrade)
+
+	var cmdParsersInspect = &cobra.Command{
+		Use:     "inspect [name]",
+		Short:   "Inspect given parser",
+		Long:    `Inspect given parser`,
+		Example: `cscli parsers inspect crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InspectItem(args[0], cwhub.PARSERS)
+		},
+	}
+	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "http://127.0.0.1:6060/metrics", "Prometheus url")
+	cmdParsers.AddCommand(cmdParsersInspect)
+
+	var cmdParsersList = &cobra.Command{
+		Use:   "list [name]",
+		Short: "List all parsers or given one",
+		Long:  `List all parsers or given one`,
+		Example: `cscli parsers list
+cscli parser list crowdsecurity/xxx`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			ListItem(cwhub.PARSERS, args)
+		},
+	}
+	cmdParsersList.PersistentFlags().BoolVarP(&listAll, "all", "a", false, "List as well disabled items")
+	cmdParsers.AddCommand(cmdParsersList)
+
+	return cmdParsers
+}

cmd/crowdsec-cli/postoverflows.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+func NewPostOverflowsCmd() *cobra.Command {
+	var cmdPostOverflows = &cobra.Command{
+		Use:   "postoverflows [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
+		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
+		cscli postoverflows inspect crowdsecurity/cdn-whitelist
+		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
+		cscli postoverflows list
+		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := setHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
+		},
+	}
+
+	var cmdPostOverflowsInstall = &cobra.Command{
+		Use:     "install [config]",
+		Short:   "Install given postoverflow(s)",
+		Long:    `Fetch and install given postoverflow(s) from hub`,
+		Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			for _, name := range args {
+				InstallItem(name, cwhub.PARSERS_OVFLW, forceInstall)
+			}
+		},
+	}
+	cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceInstall, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdPostOverflows.AddCommand(cmdPostOverflowsInstall)
+
+	var cmdPostOverflowsRemove = &cobra.Command{
+		Use:     "remove [config]",
+		Short:   "Remove given postoverflow(s)",
+		Long:    `remove given postoverflow(s)`,
+		Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			if removeAll {
+				RemoveMany(cwhub.PARSERS_OVFLW, "")
+			} else {
+				for _, name := range args {
+					RemoveMany(cwhub.PARSERS_OVFLW, name)
+				}
+			}
+		},
+	}
+	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purgeRemove, "purge", false, "Delete source file in ~/.cscli/hub/ too")
+	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&removeAll, "all", false, "Delete all the files in selected scope")
+	cmdPostOverflows.AddCommand(cmdPostOverflowsRemove)
+
+	var cmdPostOverflowsUpgrade = &cobra.Command{
+		Use:     "upgrade [config]",
+		Short:   "Upgrade given postoverflow(s)",
+		Long:    `Fetch and Upgrade given postoverflow(s) from hub`,
+		Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if upgradeAll {
+				UpgradeConfig(cwhub.PARSERS_OVFLW, "", forceUpgrade)
+			} else {
+				for _, name := range args {
+					UpgradeConfig(cwhub.PARSERS_OVFLW, name, forceUpgrade)
+				}
+			}
+		},
+	}
+	cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&upgradeAll, "download-only", "d", false, "Only download packages, don't enable")
+	cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceUpgrade, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdPostOverflows.AddCommand(cmdPostOverflowsUpgrade)
+
+	var cmdPostOverflowsInspect = &cobra.Command{
+		Use:     "inspect [config]",
+		Short:   "Inspect given postoverflow",
+		Long:    `Inspect given postoverflow`,
+		Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InspectItem(args[0], cwhub.PARSERS_OVFLW)
+		},
+	}
+	cmdPostOverflows.AddCommand(cmdPostOverflowsInspect)
+
+	var cmdPostOverflowsList = &cobra.Command{
+		Use:   "list [config]",
+		Short: "List all postoverflows or given one",
+		Long:  `List all postoverflows or given one`,
+		Example: `cscli postoverflows list
+cscli postoverflows list crowdsecurity/xxx`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			ListItem(cwhub.PARSERS_OVFLW, args)
+		},
+	}
+	cmdPostOverflowsList.PersistentFlags().BoolVarP(&listAll, "all", "a", false, "List as well disabled items")
+	cmdPostOverflows.AddCommand(cmdPostOverflowsList)
+
+	return cmdPostOverflows
+}

cmd/crowdsec-cli/remove.go

@@ -1,147 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-var purge_remove, remove_all bool
-
-func RemoveMany(ttype string, name string) {
-	var err error
-	var disabled int
-	for _, v := range cwhub.HubIdx[ttype] {
-		if name != "" && v.Name == name {
-			v, err = cwhub.DisableItem(v, cwhub.Installdir, cwhub.Hubdir, purge_remove)
-			if err != nil {
-				log.Fatalf("unable to disable %s : %v", v.Name, err)
-			}
-			cwhub.HubIdx[ttype][v.Name] = v
-			return
-		} else if name == "" && remove_all {
-			v, err = cwhub.DisableItem(v, cwhub.Installdir, cwhub.Hubdir, purge_remove)
-			if err != nil {
-				log.Fatalf("unable to disable %s : %v", v.Name, err)
-			}
-			cwhub.HubIdx[ttype][v.Name] = v
-			disabled += 1
-		}
-	}
-	if name != "" && !remove_all {
-		log.Errorf("%s not found", name)
-		return
-	}
-	log.Infof("Disabled %d items", disabled)
-}
-
-func NewRemoveCmd() *cobra.Command {
-
-	var cmdRemove = &cobra.Command{
-		Use:   "remove [type] <config>",
-		Short: "Remove/disable configuration(s)",
-		Long: `
- Remove local configuration. 
- 
-[type] must be parser, scenario, postoverflow, collection
-
-[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net) or locally installed.
- `,
-		Example: `cscli remove [type] [config_name]`,
-		Args:    cobra.MinimumNArgs(1),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
-		},
-	}
-	cmdRemove.PersistentFlags().BoolVar(&purge_remove, "purge", false, "Delete source file in ~/.cscli/hub/ too")
-	cmdRemove.PersistentFlags().BoolVar(&remove_all, "all", false, "Delete all the files in selected scope")
-	var cmdRemoveParser = &cobra.Command{
-		Use:   "parser <config>",
-		Short: "Remove/disable parser",
-		Long:  `<config> must be a valid parser.`,
-		Args:  cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-
-			if remove_all {
-				RemoveMany(cwhub.PARSERS, "")
-			} else {
-				for _, name := range args {
-					RemoveMany(cwhub.PARSERS, name)
-				}
-			}
-		},
-	}
-	cmdRemove.AddCommand(cmdRemoveParser)
-	var cmdRemoveScenario = &cobra.Command{
-		Use:   "scenario [config]",
-		Short: "Remove/disable scenario",
-		Long:  `<config> must be a valid scenario.`,
-		Args:  cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if remove_all {
-				RemoveMany(cwhub.SCENARIOS, "")
-			} else {
-				for _, name := range args {
-					RemoveMany(cwhub.SCENARIOS, name)
-				}
-			}
-		},
-	}
-	cmdRemove.AddCommand(cmdRemoveScenario)
-	var cmdRemoveCollection = &cobra.Command{
-		Use:   "collection [config]",
-		Short: "Remove/disable collection",
-		Long:  `<config> must be a valid collection.`,
-		Args:  cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if remove_all {
-				RemoveMany(cwhub.COLLECTIONS, "")
-			} else {
-				for _, name := range args {
-					RemoveMany(cwhub.COLLECTIONS, name)
-				}
-			}
-		},
-	}
-	cmdRemove.AddCommand(cmdRemoveCollection)
-
-	var cmdRemovePostoverflow = &cobra.Command{
-		Use:   "postoverflow [config]",
-		Short: "Remove/disable postoverflow parser",
-		Long:  `<config> must be a valid collection.`,
-		Args:  cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if remove_all {
-				RemoveMany(cwhub.PARSERS_OVFLW, "")
-			} else {
-				for _, name := range args {
-					RemoveMany(cwhub.PARSERS_OVFLW, name)
-				}
-			}
-		},
-	}
-	cmdRemove.AddCommand(cmdRemovePostoverflow)
-
-	return cmdRemove
-}

cmd/crowdsec-cli/scenarios.go

@@ -0,0 +1,141 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/spf13/cobra"
+)
+
+func NewScenariosCmd() *cobra.Command {
+	var cmdScenarios = &cobra.Command{
+		Use:   "scenarios [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
+		Example: `cscli scenarios list [-a]
+cscli scenarios install crowdsecurity/ssh-bf
+cscli scenarios inspect crowdsecurity/ssh-bf
+cscli scenarios upgrade crowdsecurity/ssh-bf
+cscli scenarios remove crowdsecurity/ssh-bf
+`,
+		Args: cobra.MinimumNArgs(1),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := setHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
+		},
+	}
+
+	var cmdScenariosInstall = &cobra.Command{
+		Use:     "install [config]",
+		Short:   "Install given scenario(s)",
+		Long:    `Fetch and install given scenario(s) from hub`,
+		Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			for _, name := range args {
+				InstallItem(name, cwhub.SCENARIOS, forceInstall)
+			}
+		},
+	}
+	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdScenariosInstall.PersistentFlags().BoolVar(&forceInstall, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdScenarios.AddCommand(cmdScenariosInstall)
+
+	var cmdScenariosRemove = &cobra.Command{
+		Use:     "remove [config]",
+		Short:   "Remove given scenario(s)",
+		Long:    `remove given scenario(s)`,
+		Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			if removeAll {
+				RemoveMany(cwhub.SCENARIOS, "")
+			} else {
+				for _, name := range args {
+					RemoveMany(cwhub.SCENARIOS, name)
+				}
+			}
+		},
+	}
+	cmdScenariosRemove.PersistentFlags().BoolVar(&purgeRemove, "purge", false, "Delete source file in ~/.cscli/hub/ too")
+	cmdScenariosRemove.PersistentFlags().BoolVar(&removeAll, "all", false, "Delete all the files in selected scope")
+	cmdScenarios.AddCommand(cmdScenariosRemove)
+
+	var cmdScenariosUpgrade = &cobra.Command{
+		Use:     "upgrade [config]",
+		Short:   "Upgrade given scenario(s)",
+		Long:    `Fetch and Upgrade given scenario(s) from hub`,
+		Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			if upgradeAll {
+				UpgradeConfig(cwhub.SCENARIOS, "", forceUpgrade)
+			} else {
+				for _, name := range args {
+					UpgradeConfig(cwhub.SCENARIOS, name, forceUpgrade)
+				}
+			}
+		},
+	}
+	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&upgradeAll, "download-only", "d", false, "Only download packages, don't enable")
+	cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceUpgrade, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdScenarios.AddCommand(cmdScenariosUpgrade)
+
+	var cmdScenariosInspect = &cobra.Command{
+		Use:     "inspect [config]",
+		Short:   "Inspect given scenario",
+		Long:    `Inspect given scenario`,
+		Example: `cscli scenarios inspect crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			InspectItem(args[0], cwhub.SCENARIOS)
+		},
+	}
+	cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "http://127.0.0.1:6060/metrics", "Prometheus url")
+	cmdScenarios.AddCommand(cmdScenariosInspect)
+
+	var cmdScenariosList = &cobra.Command{
+		Use:   "list [config]",
+		Short: "List all scenario(s) or given one",
+		Long:  `List all scenario(s) or given one`,
+		Example: `cscli scenarios list
+cscli scenarios list crowdsecurity/xxx`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
+				log.Fatalf("failed to get Hub index : %v", err)
+			}
+			ListItem(cwhub.SCENARIOS, args)
+		},
+	}
+	cmdScenariosList.PersistentFlags().BoolVarP(&listAll, "all", "a", false, "List as well disabled items")
+	cmdScenarios.AddCommand(cmdScenariosList)
+
+	return cmdScenarios
+}

cmd/crowdsec-cli/simulation.go

@@ -11,24 +11,25 @@ import (
 )
 
 func addToExclusion(name string) error {
-	config.SimulationCfg.Exclusions = append(config.SimulationCfg.Exclusions, name)
+	csConfig.Crowdsec.SimulationConfig.Exclusions = append(csConfig.Crowdsec.SimulationConfig.Exclusions, name)
 	return nil
 }
 
 func removeFromExclusion(name string) error {
-	index := indexOf(name, config.SimulationCfg.Exclusions)
+	index := indexOf(name, csConfig.Crowdsec.SimulationConfig.Exclusions)
 
 	// Remove element from the slice
-	config.SimulationCfg.Exclusions[index] = config.SimulationCfg.Exclusions[len(config.SimulationCfg.Exclusions)-1]
-	config.SimulationCfg.Exclusions[len(config.SimulationCfg.Exclusions)-1] = ""
-	config.SimulationCfg.Exclusions = config.SimulationCfg.Exclusions[:len(config.SimulationCfg.Exclusions)-1]
+	csConfig.Crowdsec.SimulationConfig.Exclusions[index] = csConfig.Crowdsec.SimulationConfig.Exclusions[len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1]
+	csConfig.Crowdsec.SimulationConfig.Exclusions[len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1] = ""
+	csConfig.Crowdsec.SimulationConfig.Exclusions = csConfig.Crowdsec.SimulationConfig.Exclusions[:len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1]
 
 	return nil
 }
 
 func enableGlobalSimulation() error {
-	config.SimulationCfg.Simulation = true
-	config.SimulationCfg.Exclusions = []string{}
+	csConfig.Crowdsec.SimulationConfig.Simulation = new(bool)
+	*csConfig.Crowdsec.SimulationConfig.Simulation = true
+	csConfig.Crowdsec.SimulationConfig.Exclusions = []string{}
 
 	if err := dumpSimulationFile(); err != nil {
 		log.Fatalf("unable to dump simulation file: %s", err.Error())
@@ -40,28 +41,31 @@ func enableGlobalSimulation() error {
 }
 
 func dumpSimulationFile() error {
-	newConfigSim, err := yaml.Marshal(config.SimulationCfg)
+	newConfigSim, err := yaml.Marshal(csConfig.Crowdsec.SimulationConfig)
 	if err != nil {
 		return fmt.Errorf("unable to marshal simulation configuration: %s", err)
 	}
-	err = ioutil.WriteFile(config.SimulationCfgPath, newConfigSim, 0644)
+	err = ioutil.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
 	if err != nil {
-		return fmt.Errorf("write simulation config in '%s' : %s", config.SimulationCfgPath, err)
+		return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
 	}
+	log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
 
 	return nil
 }
 
 func disableGlobalSimulation() error {
-	config.SimulationCfg.Simulation = false
-	config.SimulationCfg.Exclusions = []string{}
-	newConfigSim, err := yaml.Marshal(config.SimulationCfg)
+	csConfig.Crowdsec.SimulationConfig.Simulation = new(bool)
+	*csConfig.Crowdsec.SimulationConfig.Simulation = false
+
+	csConfig.Crowdsec.SimulationConfig.Exclusions = []string{}
+	newConfigSim, err := yaml.Marshal(csConfig.Crowdsec.SimulationConfig)
 	if err != nil {
 		return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
 	}
-	err = ioutil.WriteFile(config.SimulationCfgPath, newConfigSim, 0644)
+	err = ioutil.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
 	if err != nil {
-		return fmt.Errorf("unable to write new simulation config in '%s' : %s", config.SimulationCfgPath, err)
+		return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
 	}
 
 	log.Printf("global simulation: disabled")
@@ -69,23 +73,23 @@ func disableGlobalSimulation() error {
 }
 
 func simulationStatus() error {
-	if config.SimulationCfg == nil {
+	if csConfig.Crowdsec.SimulationConfig == nil {
 		log.Printf("global simulation: disabled (configuration file is missing)")
 		return nil
 	}
-	if config.SimulationCfg.Simulation {
+	if *csConfig.Crowdsec.SimulationConfig.Simulation {
 		log.Println("global simulation: enabled")
-		if len(config.SimulationCfg.Exclusions) > 0 {
+		if len(csConfig.Crowdsec.SimulationConfig.Exclusions) > 0 {
 			log.Println("Scenarios not in simulation mode :")
-			for _, scenario := range config.SimulationCfg.Exclusions {
+			for _, scenario := range csConfig.Crowdsec.SimulationConfig.Exclusions {
 				log.Printf("  - %s", scenario)
 			}
 		}
 	} else {
 		log.Println("global simulation: disabled")
-		if len(config.SimulationCfg.Exclusions) > 0 {
+		if len(csConfig.Crowdsec.SimulationConfig.Exclusions) > 0 {
 			log.Println("Scenarios in simulation mode :")
-			for _, scenario := range config.SimulationCfg.Exclusions {
+			for _, scenario := range csConfig.Crowdsec.SimulationConfig.Exclusions {
 				log.Printf("  - %s", scenario)
 			}
 		}
@@ -95,55 +99,59 @@ func simulationStatus() error {
 
 func NewSimulationCmds() *cobra.Command {
 	var cmdSimulation = &cobra.Command{
-		Use:   "simulation enable|disable [scenario_name]",
-		Short: "",
-		Long:  ``,
+		Use:   "simulation [command]",
+		Short: "Manage simulation status of scenarios",
+		Example: `cscli simulation status
+cscli simulation enable crowdsecurity/ssh-bf
+cscli simulation disable crowdsecurity/ssh-bf`,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
+			if csConfig.Cscli == nil {
 				return fmt.Errorf("you must configure cli before using simulation")
 			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
+			if cmd.Name() != "status" {
+				log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
+			}
 		},
 	}
 	cmdSimulation.Flags().SortFlags = false
 	cmdSimulation.PersistentFlags().SortFlags = false
 
+	var forceGlobalSimulation bool
 	var cmdSimulationEnable = &cobra.Command{
-		Use:     "enable [scenario_name]",
+		Use:     "enable [scenario] [-global]",
 		Short:   "Enable the simulation, globally or on specified scenarios",
-		Long:    ``,
 		Example: `cscli simulation enable`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
+			if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
 				log.Fatalf("failed to get Hub index : %v", err)
 			}
 
 			if len(args) > 0 {
 				for _, scenario := range args {
-					var v cwhub.Item
-					var ok bool
-					if _, ok = cwhub.HubIdx[cwhub.SCENARIOS]; ok {
-						if v, ok = cwhub.HubIdx[cwhub.SCENARIOS][scenario]; !ok {
-							log.Errorf("'%s' isn't present in hub index", scenario)
-							continue
-						}
-						if !v.Installed {
-							log.Warningf("'%s' isn't enabled", scenario)
-						}
+					var (
+						item *cwhub.Item
+					)
+					item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
+					if item == nil {
+						log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
+						continue
 					}
-					isExcluded := inSlice(scenario, config.SimulationCfg.Exclusions)
-					if config.SimulationCfg.Simulation && !isExcluded {
+					if !item.Installed {
+						log.Warningf("'%s' isn't enabled", scenario)
+					}
+					isExcluded := inSlice(scenario, csConfig.Crowdsec.SimulationConfig.Exclusions)
+					if *csConfig.Crowdsec.SimulationConfig.Simulation && !isExcluded {
 						log.Warningf("global simulation is already enabled")
 						continue
 					}
-					if !config.SimulationCfg.Simulation && isExcluded {
+					if !*csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
 						log.Warningf("simulation for '%s' already enabled", scenario)
 						continue
 					}
-					if config.SimulationCfg.Simulation && isExcluded {
+					if *csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
 						if err := removeFromExclusion(scenario); err != nil {
 							log.Fatalf(err.Error())
 						}
@@ -158,29 +166,31 @@ func NewSimulationCmds() *cobra.Command {
 				if err := dumpSimulationFile(); err != nil {
 					log.Fatalf("simulation enable: %s", err.Error())
 				}
-			} else {
+			} else if forceGlobalSimulation {
 				if err := enableGlobalSimulation(); err != nil {
 					log.Fatalf("unable to enable global simulation mode : %s", err.Error())
 				}
+			} else {
+				cmd.Help()
 			}
 		},
 	}
+	cmdSimulationEnable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
 	cmdSimulation.AddCommand(cmdSimulationEnable)
 
 	var cmdSimulationDisable = &cobra.Command{
-		Use:     "disable [scenario_name]",
+		Use:     "disable [scenario]",
 		Short:   "Disable the simulation mode. Disable only specified scenarios",
-		Long:    ``,
 		Example: `cscli simulation disable`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if len(args) > 0 {
 				for _, scenario := range args {
-					isExcluded := inSlice(scenario, config.SimulationCfg.Exclusions)
-					if !config.SimulationCfg.Simulation && !isExcluded {
+					isExcluded := inSlice(scenario, csConfig.Crowdsec.SimulationConfig.Exclusions)
+					if !*csConfig.Crowdsec.SimulationConfig.Simulation && !isExcluded {
 						log.Warningf("%s isn't in simulation mode", scenario)
 						continue
 					}
-					if !config.SimulationCfg.Simulation && isExcluded {
+					if !*csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
 						if err := removeFromExclusion(scenario); err != nil {
 							log.Fatalf(err.Error())
 						}
@@ -199,19 +209,21 @@ func NewSimulationCmds() *cobra.Command {
 				if err := dumpSimulationFile(); err != nil {
 					log.Fatalf("simulation disable: %s", err.Error())
 				}
-			} else {
+			} else if forceGlobalSimulation {
 				if err := disableGlobalSimulation(); err != nil {
 					log.Fatalf("unable to disable global simulation mode : %s", err.Error())
 				}
+			} else {
+				cmd.Help()
 			}
 		},
 	}
+	cmdSimulationDisable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
 	cmdSimulation.AddCommand(cmdSimulationDisable)
 
 	var cmdSimulationStatus = &cobra.Command{
 		Use:     "status",
 		Short:   "Show simulation mode status",
-		Long:    ``,
 		Example: `cscli simulation status`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := simulationStatus(); err != nil {

cmd/crowdsec-cli/update.go

@@ -1,38 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-func NewUpdateCmd() *cobra.Command {
-	/* ---- UPDATE COMMAND */
-	var cmdUpdate = &cobra.Command{
-		Use:   "update",
-		Short: "Fetch available configs from hub",
-		Long: `
-Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
-`,
-		Args: cobra.ExactArgs(0),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := setHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-			return nil
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.UpdateHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-		},
-	}
-	return cmdUpdate
-}

cmd/crowdsec-cli/upgrade.go

@@ -1,214 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-
-	"github.com/enescakir/emoji"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-var upgrade_all, force_upgrade bool
-
-func UpgradeConfig(ttype string, name string) {
-	var err error
-	var updated int
-	var found bool
-
-	for _, v := range cwhub.HubIdx[ttype] {
-		//name mismatch
-		if name != "" && name != v.Name {
-			continue
-		}
-		if !v.Installed {
-			log.Debugf("skip %s, not installed", v.Name)
-			continue
-		}
-		if !v.Downloaded {
-			log.Warningf("%s : not downloaded, please install.", v.Name)
-			continue
-		}
-		found = true
-		if v.UpToDate && !force_upgrade {
-			log.Infof("%s : up-to-date", v.Name)
-			continue
-		}
-		v, err = cwhub.DownloadLatest(v, cwhub.Hubdir, force_upgrade, config.DataFolder)
-		if err != nil {
-			log.Fatalf("%s : download failed : %v", v.Name, err)
-		}
-		if !v.UpToDate {
-			if v.Tainted && !force_upgrade {
-				log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
-				continue
-			} else if v.Local {
-				log.Infof("%v %s is local", emoji.Prohibited, v.Name)
-				continue
-			}
-		} else {
-			log.Infof("%v %s : updated", emoji.Package, v.Name)
-			updated += 1
-		}
-		cwhub.HubIdx[ttype][v.Name] = v
-	}
-	if !found {
-		log.Errorf("Didn't find %s", name)
-	} else if updated == 0 && found {
-		log.Errorf("Nothing to update")
-	} else if updated != 0 {
-		log.Infof("Upgraded %d items", updated)
-	}
-
-}
-
-func NewUpgradeCmd() *cobra.Command {
-
-	var cmdUpgrade = &cobra.Command{
-		Use:   "upgrade [type] [config]",
-		Short: "Upgrade configuration(s)",
-		Long: `
-Upgrade configuration from the CrowdSec Hub.
-
-In order to upgrade latest versions of configuration, 
-the Hub cache should be [updated](./cscli_update.md).
- 
-Tainted configuration will not be updated (use --force to update them).
-
-[type] must be parser, scenario, postoverflow, collection.
-
-[config_name] must be a valid config name from [Crowdsec Hub](https://hub.crowdsec.net).
- 
-
- `,
-		Example: `cscli upgrade [type] [config_name]
-cscli upgrade --all   # Upgrade all configurations types
-cscli upgrade --force # Overwrite tainted configuration
-		`,
-
-		Args: cobra.MinimumNArgs(0),
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if !config.configured {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := setHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-			return nil
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if !upgrade_all && len(args) < 2 {
-				_ = cmd.Help()
-				return
-			}
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if upgrade_all && len(args) == 0 {
-				log.Warningf("Upgrade all : parsers, scenarios, collections.")
-				UpgradeConfig(cwhub.PARSERS, "")
-				UpgradeConfig(cwhub.PARSERS_OVFLW, "")
-				UpgradeConfig(cwhub.SCENARIOS, "")
-				UpgradeConfig(cwhub.COLLECTIONS, "")
-			}
-			//fmt.Println("upgrade all ?!: " + strings.Join(args, " "))
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
-		},
-	}
-	cmdUpgrade.PersistentFlags().BoolVar(&upgrade_all, "all", false, "Upgrade all configuration in scope")
-	cmdUpgrade.PersistentFlags().BoolVar(&force_upgrade, "force", false, "Overwrite existing files, even if tainted")
-	var cmdUpgradeParser = &cobra.Command{
-		Use:   "parser [config]",
-		Short: "Upgrade parser configuration(s)",
-		Long:  `Upgrade one or more parser configurations`,
-		Example: ` - cscli upgrade parser crowdsec/apache-logs  
- - cscli upgrade parser -all  
- - cscli upgrade parser crowdsec/apache-logs --force`,
-		Args: cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if upgrade_all {
-				UpgradeConfig(cwhub.PARSERS, "")
-			} else {
-				for _, name := range args {
-					UpgradeConfig(cwhub.PARSERS, name)
-				}
-			}
-
-		},
-	}
-	cmdUpgrade.AddCommand(cmdUpgradeParser)
-	var cmdUpgradeScenario = &cobra.Command{
-		Use:   "scenario [config]",
-		Short: "Upgrade scenario configuration(s)",
-		Long:  `Upgrade one or more scenario configurations`,
-		Example: ` - cscli	upgrade scenario -all  
- - cscli upgrade scenario crowdsec/http-404 --force  `,
-		Args: cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if upgrade_all {
-				UpgradeConfig(cwhub.SCENARIOS, "")
-			} else {
-				for _, name := range args {
-					UpgradeConfig(cwhub.SCENARIOS, name)
-				}
-			}
-		},
-	}
-	cmdUpgrade.AddCommand(cmdUpgradeScenario)
-	var cmdUpgradeCollection = &cobra.Command{
-		Use:   "collection [config]",
-		Short: "Upgrade collection configuration(s)",
-		Long:  `Upgrade one or more collection configurations`,
-		Example: ` - cscli upgrade collection crowdsec/apache-lamp  
- - cscli upgrade collection -all  
- - cscli upgrade collection crowdsec/apache-lamp --force`,
-		Args: cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if upgrade_all {
-				UpgradeConfig(cwhub.COLLECTIONS, "")
-			} else {
-				for _, name := range args {
-					UpgradeConfig(cwhub.COLLECTIONS, name)
-				}
-			}
-		},
-	}
-	cmdUpgrade.AddCommand(cmdUpgradeCollection)
-
-	var cmdUpgradePostoverflow = &cobra.Command{
-		Use:   "postoverflow [config]",
-		Short: "Upgrade postoverflow parser configuration(s)",
-		Long:  `Upgrade one or more postoverflow parser configurations`,
-		Example: ` - cscli upgrade postoverflow crowdsec/enrich-rdns  
- - cscli upgrade postoverflow -all  
- - cscli upgrade postoverflow crowdsec/enrich-rdns --force`,
-		Args: cobra.MinimumNArgs(0),
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := cwhub.GetHubIdx(); err != nil {
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			if upgrade_all {
-				UpgradeConfig(cwhub.PARSERS_OVFLW, "")
-			} else {
-				for _, name := range args {
-					UpgradeConfig(cwhub.PARSERS_OVFLW, name)
-				}
-			}
-		},
-	}
-	cmdUpgrade.AddCommand(cmdUpgradePostoverflow)
-	return cmdUpgrade
-}

cmd/crowdsec-cli/utils.go

@@ -1,10 +1,26 @@
 package main
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/enescakir/emoji"
+	"github.com/olekukonko/tablewriter"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/mod/semver"
+	"gopkg.in/yaml.v2"
 )
 
 func inSlice(s string, slice []string) bool {
@@ -25,6 +41,32 @@ func indexOf(s string, slice []string) int {
 	return -1
 }
 
+func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
+
+	/*if a range is provided, change the scope*/
+	if *ipRange != "" {
+		_, _, err := net.ParseCIDR(*ipRange)
+		if err != nil {
+			return fmt.Errorf("%s isn't a valid range", *ipRange)
+		}
+	}
+	if *ip != "" {
+		ipRepr := net.ParseIP(*ip)
+		if ipRepr == nil {
+			return fmt.Errorf("%s isn't a valid ip", *ip)
+		}
+	}
+
+	//avoid confusion on scope (ip vs Ip and range vs Range)
+	switch strings.ToLower(*scope) {
+	case "ip":
+		*scope = types.Ip
+	case "range":
+		*scope = types.Range
+	}
+	return nil
+}
+
 func setHubBranch() error {
 	/*
 		if no branch has been specified in flags for the hub, then use the one corresponding to crowdsec version
@@ -51,3 +93,549 @@ func setHubBranch() error {
 	}
 	return nil
 }
+
+func ListItem(itemType string, args []string) {
+
+	var hubStatus []map[string]string
+
+	if len(args) == 1 {
+		hubStatus = cwhub.HubStatus(itemType, args[0], listAll)
+	} else {
+		hubStatus = cwhub.HubStatus(itemType, "", listAll)
+	}
+
+	if csConfig.Cscli.Output == "human" {
+
+		table := tablewriter.NewWriter(os.Stdout)
+		table.SetCenterSeparator("")
+		table.SetColumnSeparator("")
+
+		table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+		table.SetAlignment(tablewriter.ALIGN_LEFT)
+		table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
+		for _, v := range hubStatus {
+			table.Append([]string{v["name"], v["utf8_status"], v["local_version"], v["local_path"]})
+		}
+		table.Render()
+	} else if csConfig.Cscli.Output == "json" {
+		x, err := json.MarshalIndent(hubStatus, "", " ")
+		if err != nil {
+			log.Fatalf("failed to unmarshal")
+		}
+		fmt.Printf("%s", string(x))
+	} else if csConfig.Cscli.Output == "raw" {
+		for _, v := range hubStatus {
+			fmt.Printf("%s %s\n", v["name"], v["description"])
+		}
+	}
+}
+
+func InstallItem(name string, obtype string, force bool) {
+	it := cwhub.GetItem(obtype, name)
+	if it == nil {
+		log.Fatalf("unable to retrive item : %s", name)
+	}
+	item := *it
+	if downloadOnly && item.Downloaded && item.UpToDate {
+		log.Warningf("%s is already downloaded and up-to-date", item.Name)
+		if !force {
+			return
+		}
+	}
+	item, err := cwhub.DownloadLatest(csConfig.Cscli, item, forceInstall)
+	if err != nil {
+		log.Fatalf("error while downloading %s : %v", item.Name, err)
+	}
+	cwhub.AddItem(obtype, item)
+	if downloadOnly {
+		log.Infof("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath)
+		return
+	}
+	item, err = cwhub.EnableItem(csConfig.Cscli, item)
+	if err != nil {
+		log.Fatalf("error while enabled %s : %v.", item.Name, err)
+	}
+	cwhub.AddItem(obtype, item)
+	log.Infof("Enabled %s", item.Name)
+	return
+}
+
+func RemoveMany(itemType string, name string) {
+	var err error
+	var disabled int
+	if name != "" {
+		it := cwhub.GetItem(itemType, name)
+		if it == nil {
+			log.Fatalf("unable to retrieve: %s", name)
+		}
+		item := *it
+		item, err = cwhub.DisableItem(csConfig.Cscli, item, purgeRemove)
+		if err != nil {
+			log.Fatalf("unable to disable %s : %v", item.Name, err)
+		}
+		cwhub.AddItem(itemType, item)
+		return
+	} else if name == "" && removeAll {
+		for _, v := range cwhub.GetItemMap(itemType) {
+			v, err = cwhub.DisableItem(csConfig.Cscli, v, purgeRemove)
+			if err != nil {
+				log.Fatalf("unable to disable %s : %v", v.Name, err)
+			}
+			cwhub.AddItem(itemType, v)
+			disabled++
+		}
+	}
+	if name != "" && !removeAll {
+		log.Errorf("%s not found", name)
+		return
+	}
+	log.Infof("Disabled %d items", disabled)
+}
+
+func UpgradeConfig(itemType string, name string, force bool) {
+	var err error
+	var updated int
+	var found bool
+
+	for _, v := range cwhub.GetItemMap(itemType) {
+		if name != "" && name != v.Name {
+			continue
+		}
+		if !v.Installed {
+			log.Tracef("skip %s, not installed", v.Name)
+			if !force {
+				continue
+			}
+		}
+		if !v.Downloaded {
+			log.Warningf("%s : not downloaded, please install.", v.Name)
+			if !force {
+				continue
+			}
+		}
+		found = true
+		if v.UpToDate {
+			log.Infof("%s : up-to-date", v.Name)
+			if !force {
+				continue
+			}
+		}
+		v, err = cwhub.DownloadLatest(csConfig.Cscli, v, forceUpgrade)
+		if err != nil {
+			log.Fatalf("%s : download failed : %v", v.Name, err)
+		}
+		if !v.UpToDate {
+			if v.Tainted {
+				log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
+			} else if v.Local {
+				log.Infof("%v %s is local", emoji.Prohibited, v.Name)
+			}
+		} else {
+			log.Infof("%v %s : updated", emoji.Package, v.Name)
+			updated++
+		}
+		cwhub.AddItem(itemType, v)
+	}
+	if !found {
+		log.Errorf("Didn't find %s", name)
+	} else if updated == 0 && found {
+		log.Errorf("Nothing to update")
+	} else if updated != 0 {
+		log.Infof("Upgraded %d items", updated)
+	}
+
+}
+
+func InspectItem(name string, objecitemType string) {
+
+	hubItem := cwhub.GetItem(objecitemType, name)
+	if hubItem == nil {
+		log.Fatalf("unable to retrieve item.")
+	}
+	buff, err := yaml.Marshal(*hubItem)
+	if err != nil {
+		log.Fatalf("unable to marshal item : %s", err)
+	}
+	fmt.Printf("%s", string(buff))
+
+	fmt.Printf("\nCurrent metrics : \n\n")
+	ShowMetrics(hubItem)
+
+}
+
+func ShowMetrics(hubItem *cwhub.Item) {
+	switch hubItem.Type {
+	case cwhub.PARSERS:
+		metrics := GetParserMetric(prometheusURL, hubItem.Name)
+		ShowParserMetric(hubItem.Name, metrics)
+	case cwhub.SCENARIOS:
+		metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
+		ShowScenarioMetric(hubItem.Name, metrics)
+	case cwhub.COLLECTIONS:
+		for _, item := range hubItem.Parsers {
+			metrics := GetParserMetric(prometheusURL, item)
+			ShowParserMetric(item, metrics)
+		}
+		for _, item := range hubItem.Scenarios {
+			metrics := GetScenarioMetric(prometheusURL, item)
+			ShowScenarioMetric(item, metrics)
+		}
+		for _, item := range hubItem.Collections {
+			hubItem := cwhub.GetItem(cwhub.COLLECTIONS, item)
+			if hubItem == nil {
+				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
+			}
+			ShowMetrics(hubItem)
+		}
+	default:
+		log.Errorf("item of type '%s' is unknown", hubItem.Type)
+	}
+}
+
+/*This is a complete rip from prom2json*/
+func GetParserMetric(url string, itemName string) map[string]map[string]int {
+	stats := make(map[string]map[string]int)
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric := m.(prom2json.Metric)
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric %v", metric.Labels)
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_reader_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+					stats[source]["parsed"] = 0
+					stats[source]["reads"] = 0
+					stats[source]["unparsed"] = 0
+					stats[source]["hits"] = 0
+				}
+				stats[source]["reads"] += ival
+			case "cs_parser_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			case "cs_node_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["hits"] += ival
+			case "cs_node_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_node_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetScenarioMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["instanciation"] = 0
+	stats["curr_count"] = 0
+	stats["overflow"] = 0
+	stats["pour"] = 0
+	stats["underflow"] = 0
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric := m.(prom2json.Metric)
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_bucket_created_total":
+				stats["instanciation"] += ival
+			case "cs_buckets":
+				stats["curr_count"] += ival
+			case "cs_bucket_overflowed_total":
+				stats["overflow"] += ival
+			case "cs_bucket_poured_total":
+				stats["pour"] += ival
+			case "cs_bucket_underflowed_total":
+				stats["underflow"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetPrometheusMetric(url string) []*prom2json.Family {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		defer types.CatchPanic("crowdsec/GetPrometheusMetric")
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+
+	return result
+}
+
+func ShowScenarioMetric(itemName string, metrics map[string]int) {
+	if metrics["instanciation"] == 0 {
+		return
+	}
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Current Count", "Overflows", "Instanciated", "Poured", "Expired"})
+	table.Append([]string{fmt.Sprintf("%d", metrics["curr_count"]), fmt.Sprintf("%d", metrics["overflow"]), fmt.Sprintf("%d", metrics["instanciation"]), fmt.Sprintf("%d", metrics["pour"]), fmt.Sprintf("%d", metrics["underflow"])})
+
+	fmt.Printf(" - (Scenario) %s: \n", itemName)
+	table.Render()
+	fmt.Println()
+}
+
+func ShowParserMetric(itemName string, metrics map[string]map[string]int) {
+	skip := true
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Parsers", "Hits", "Parsed", "Unparsed"})
+	for source, stats := range metrics {
+		if stats["hits"] > 0 {
+			table.Append([]string{source, fmt.Sprintf("%d", stats["hits"]), fmt.Sprintf("%d", stats["parsed"]), fmt.Sprintf("%d", stats["unparsed"])})
+			skip = false
+		}
+	}
+	if !skip {
+		fmt.Printf(" - (Parser) %s: \n", itemName)
+		table.Render()
+		fmt.Println()
+	}
+}
+
+//it's a rip of the cli version, but in silent-mode
+func silenceInstallItem(name string, obtype string) (string, error) {
+	var item *cwhub.Item
+	item = cwhub.GetItem(obtype, name)
+	if item == nil {
+		return "", fmt.Errorf("error retrieving item")
+	}
+	it := *item
+	if downloadOnly && it.Downloaded && it.UpToDate {
+		return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
+	}
+	it, err := cwhub.DownloadLatest(csConfig.Cscli, it, forceInstall)
+	if err != nil {
+		return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, it); err != nil {
+		return "", err
+	}
+
+	if downloadOnly {
+		return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
+	}
+	it, err = cwhub.EnableItem(csConfig.Cscli, it)
+	if err != nil {
+		return "", fmt.Errorf("error while enabled %s : %v", it.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, it); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("Enabled %s", it.Name), nil
+}
+
+func RestoreHub(dirPath string) error {
+	var err error
+
+	for _, itype := range cwhub.ItemTypes {
+		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
+		if _, err = os.Stat(itemDirectory); err != nil {
+			log.Infof("no %s in backup", itype)
+			continue
+		}
+		/*restore the upstream items*/
+		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
+		file, err := ioutil.ReadFile(upstreamListFN)
+		if err != nil {
+			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
+		}
+		var upstreamList []string
+		err = json.Unmarshal([]byte(file), &upstreamList)
+		if err != nil {
+			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
+		}
+		for _, toinstall := range upstreamList {
+			label, err := silenceInstallItem(toinstall, itype)
+			if err != nil {
+				log.Errorf("Error while installing %s : %s", toinstall, err)
+			} else if label != "" {
+				log.Infof("Installed %s : %s", toinstall, label)
+			} else {
+				log.Printf("Installed %s : ok", toinstall)
+			}
+		}
+		/*restore the local and tainted items*/
+		files, err := ioutil.ReadDir(itemDirectory)
+		if err != nil {
+			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
+		}
+		for _, file := range files {
+			//dir are stages, keep track
+			if !file.IsDir() {
+				continue
+			}
+			stage := file.Name()
+			stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
+			log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
+			if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
+				return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
+			}
+			/*find items*/
+			ifiles, err := ioutil.ReadDir(itemDirectory + "/" + stage + "/")
+			if err != nil {
+				return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
+			}
+			//finaly copy item
+			for _, tfile := range ifiles {
+				log.Infof("Going to restore local/tainted [%s]", tfile.Name())
+				sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
+				destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
+				if err = types.CopyFile(sourceFile, destinationFile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+				}
+				log.Infof("restored %s to %s", sourceFile, destinationFile)
+			}
+		}
+	}
+	return nil
+}
+
+func BackupHub(dirPath string) error {
+	var err error
+	var itemDirectory string
+	var upstreamParsers []string
+
+	for _, itemType := range cwhub.ItemTypes {
+		clog := log.WithFields(log.Fields{
+			"type": itemType,
+		})
+		itemMap := cwhub.GetItemMap(itemType)
+		if itemMap != nil {
+			itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
+			if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+				return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
+			}
+			upstreamParsers = []string{}
+			for k, v := range itemMap {
+				clog = clog.WithFields(log.Fields{
+					"file": v.Name,
+				})
+				if !v.Installed { //only backup installed ones
+					clog.Debugf("[%s] : not installed", k)
+					continue
+				}
+
+				//for the local/tainted ones, we backup the full file
+				if v.Tainted || v.Local || !v.UpToDate {
+					//we need to backup stages for parsers
+					if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+						fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
+						if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+							return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
+						}
+					}
+					clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
+					tfile := fmt.Sprintf("%s%s%s", itemDirectory, v.Stage, v.FileName)
+					if err = types.CopyFile(v.LocalPath, tfile); err != nil {
+						return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
+					}
+					clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
+					continue
+				}
+				clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
+				clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
+				upstreamParsers = append(upstreamParsers, v.Name)
+			}
+			//write the upstream items
+			upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
+			upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
+			if err != nil {
+				return fmt.Errorf("failed marshaling upstream parsers : %s", err)
+			}
+			err = ioutil.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
+			if err != nil {
+				return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
+			}
+			clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
+
+		} else {
+			clog.Infof("No %s to backup.", itemType)
+		}
+	}
+
+	return nil
+}

cmd/crowdsec/api.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/apiserver"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+)
+
+func initAPIServer() (*apiserver.APIServer, error) {
+	apiServer, err := apiserver.NewServer(cConfig.API.Server)
+	if err != nil {
+		return nil, fmt.Errorf("unable to run local API: %s", err)
+	}
+
+	return apiServer, nil
+}
+
+func serveAPIServer(apiServer *apiserver.APIServer) {
+	apiTomb.Go(func() error {
+		defer types.CatchPanic("crowdsec/serveAPIServer")
+		go func() {
+			defer types.CatchPanic("crowdsec/runAPIServer")
+			if err := apiServer.Run(); err != nil {
+				log.Fatalf(err.Error())
+			}
+		}()
+		<-apiTomb.Dying() // lock until go routine is dying
+		log.Infof("serve: shutting down api server")
+		if err := apiServer.Shutdown(); err != nil {
+			return err
+		}
+		return nil
+	})
+}

cmd/crowdsec/crowdsec.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
+	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
+	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+)
+
+func initCrowdsec() (*parser.Parsers, error) {
+	err := exprhelpers.Init()
+	if err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Failed to init expr helpers : %s", err)
+	}
+
+	// Populate cwhub package tools
+	if err := cwhub.GetHubIdx(cConfig.Cscli); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
+	}
+
+	// Start loading configs
+	csParsers := newParsers()
+	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
+	}
+
+	if err := LoadBuckets(cConfig); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
+	}
+
+	if err := LoadAcquisition(cConfig); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
+	}
+	return csParsers, nil
+}
+
+func runCrowdsec(parsers *parser.Parsers) error {
+	inputLineChan := make(chan types.Event)
+	inputEventChan := make(chan types.Event)
+
+	//start go-routines for parsing, buckets pour and ouputs.
+	for i := 0; i < cConfig.Crowdsec.ParserRoutinesCount; i++ {
+		parsersTomb.Go(func() error {
+			defer types.CatchPanic("crowdsec/runParse")
+			err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes)
+			if err != nil {
+				log.Fatalf("starting parse error : %s", err)
+				return err
+			}
+			return nil
+		})
+	}
+
+	for i := 0; i < cConfig.Crowdsec.BucketsRoutinesCount; i++ {
+		bucketsTomb.Go(func() error {
+			defer types.CatchPanic("crowdsec/runPour")
+			err := runPour(inputEventChan, holders, buckets)
+			if err != nil {
+				log.Fatalf("starting pour error : %s", err)
+				return err
+			}
+			return nil
+		})
+	}
+	for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
+
+		outputsTomb.Go(func() error {
+			defer types.CatchPanic("crowdsec/runOutput")
+			err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials)
+			if err != nil {
+				log.Fatalf("starting outputs error : %s", err)
+				return err
+			}
+			return nil
+		})
+	}
+	log.Warningf("Starting processing data")
+
+	if err := acquisition.StartAcquisition(dataSources, inputLineChan, &acquisTomb); err != nil {
+		log.Fatalf("starting acquisition error : %s", err)
+		return err
+	}
+
+	return nil
+}
+
+func serveCrowdsec(parsers *parser.Parsers) {
+	crowdsecTomb.Go(func() error {
+		defer types.CatchPanic("crowdsec/serveCrowdsec")
+		go func() {
+			defer types.CatchPanic("crowdsec/runCrowdsec")
+			runCrowdsec(parsers)
+		}()
+
+		/*we should stop in two cases :
+		- crowdsecTomb has been Killed() : it might be shutdown or reload, so stop
+		- acquisTomb is dead, it means that we were in "cat" mode and files are done reading, quit
+		*/
+		waitOnTomb()
+		log.Debugf("Shutting down crowdsec routines")
+		if err := ShutdownCrowdsecRoutines(); err != nil {
+			log.Fatalf("unable to shutdown crowdsec routines: %s", err)
+		}
+		log.Debugf("everything is dead, return crowdsecTomb")
+		return nil
+	})
+}
+
+func waitOnTomb() {
+	for {
+		select {
+		case <-acquisTomb.Dead():
+			/*if it's acquisition dying it means that we were in "cat" mode.
+			while shutting down, we need to give time for all buckets to process in flight data*/
+			log.Warningf("Acquisition is finished, shutting down")
+			bucketCount := leaky.LeakyRoutineCount
+			rounds := 0
+			successiveStillRounds := 0
+			/*
+				While it might make sense to want to shut-down parser/buckets/etc. as soon as acquisition is finished,
+				we might have some pending buckets : buckets that overflowed, but which LeakRoutine are still alive because they
+				are waiting to be able to "commit" (push to api). This can happens specifically in a context where a lot of logs
+				are going to trigger overflow (ie. trigger buckets with ~100% of the logs triggering an overflow).
+
+				To avoid this (which would mean that we would "lose" some overflows), let's monitor the number of live buckets.
+				However, because of the blackhole mechanism, you can't really wait for the number of LeakRoutine to go to zero (we might have to wait $blackhole_duration).
+
+				So : we are waiting for the number of buckets to stop decreasing before returning. "how long" we should wait is a bit of the trick question,
+				as some operations (ie. reverse dns or such in post-overflow) can take some time :)
+			*/
+			for {
+				currBucketCount := leaky.LeakyRoutineCount
+
+				if currBucketCount == 0 {
+					/*no bucket to wait on*/
+					break
+				}
+				if currBucketCount != bucketCount {
+					if rounds == 0 || rounds%2 == 0 {
+						log.Printf("Still %d live LeakRoutines, waiting (was %d)", currBucketCount, bucketCount)
+					}
+					bucketCount = currBucketCount
+					successiveStillRounds = 0
+				} else {
+					if successiveStillRounds > 1 {
+						log.Printf("LeakRoutines commit over.")
+						break
+					}
+					successiveStillRounds++
+				}
+				rounds++
+				time.Sleep(5 * time.Second)
+			}
+			return
+		case <-crowdsecTomb.Dying():
+			log.Infof("Crowdsec engine shutting down")
+			return
+		}
+	}
+}

cmd/crowdsec/main.go

@@ -1,22 +1,23 @@
 package main
 
 import (
+	"flag"
 	"fmt"
-	"syscall"
+	"os"
+	"strings"
 
 	_ "net/http/pprof"
 	"time"
 
+	"sort"
+
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
-	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
-	"github.com/crowdsecurity/crowdsec/pkg/outputs"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-	"github.com/pkg/errors"
-	"github.com/sevlyar/go-daemon"
 
 	log "github.com/sirupsen/logrus"
 
@@ -25,102 +26,108 @@ import (
 
 var (
 	/*tombs for the parser, buckets and outputs.*/
-	acquisTomb  tomb.Tomb
-	parsersTomb tomb.Tomb
-	bucketsTomb tomb.Tomb
-	outputsTomb tomb.Tomb
+	acquisTomb   tomb.Tomb
+	parsersTomb  tomb.Tomb
+	bucketsTomb  tomb.Tomb
+	outputsTomb  tomb.Tomb
+	apiTomb      tomb.Tomb
+	crowdsecTomb tomb.Tomb
+
+	disableAPI   bool
+	disableAgent bool
+
+	flags *Flags
+
 	/*global crowdsec config*/
-	cConfig *csconfig.CrowdSec
+	cConfig *csconfig.GlobalConfig
 	/*the state of acquisition*/
-	acquisitionCTX *acquisition.FileAcquisCtx
+	dataSources []acquisition.DataSource
 	/*the state of the buckets*/
 	holders         []leaky.BucketFactory
 	buckets         *leaky.Buckets
 	outputEventChan chan types.Event //the buckets init returns its own chan that is used for multiplexing
-	/*the state of outputs*/
-	OutputRunner   *outputs.Output
-	outputProfiles []types.Profile
-	/*the state of the parsers*/
-	parserCTX         *parser.UnixParserCtx
-	postOverflowCTX   *parser.UnixParserCtx
-	parserNodes       []parser.Node
-	postOverflowNodes []parser.Node
 	/*settings*/
 	lastProcessedItem time.Time /*keep track of last item timestamp in time-machine. it is used to GC buckets when we dump them.*/
 )
 
-func LoadParsers(cConfig *csconfig.CrowdSec) error {
-	var p parser.UnixParser
-	var err error
-
-	parserNodes = make([]parser.Node, 0)
-	postOverflowNodes = make([]parser.Node, 0)
-
-	log.Infof("Loading grok library")
-	/* load base regexps for two grok parsers */
-	parserCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/"), "data": cConfig.DataFolder})
-	if err != nil {
-		return fmt.Errorf("failed to load parser patterns : %v", err)
-	}
-	postOverflowCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/"), "data": cConfig.DataFolder})
-	if err != nil {
-		return fmt.Errorf("failed to load postovflw parser patterns : %v", err)
-	}
-
-	/*
-		Load enrichers
-	*/
-	log.Infof("Loading enrich plugins")
-	parserPlugins, err := parser.Loadplugin(cConfig.DataFolder)
-	if err != nil {
-		return fmt.Errorf("Failed to load enrich plugin : %v", err)
-	}
-	parser.ECTX = []parser.EnricherCtx{parserPlugins}
-
-	/*
-	 Load the actual parsers
-	*/
+type Flags struct {
+	ConfigFile             string
+	TraceLevel             bool
+	DebugLevel             bool
+	InfoLevel              bool
+	PrintVersion           bool
+	SingleFilePath         string
+	SingleJournalctlFilter string
+	SingleFileType         string
+	SingleFileJsonOutput   string
+	TestMode               bool
+	DisableAgent           bool
+	DisableAPI             bool
+}
 
-	log.Infof("Loading parsers")
-	parserNodes, err = parser.LoadStageDir(cConfig.ConfigFolder+"/parsers/", parserCTX)
+type parsers struct {
+	ctx             *parser.UnixParserCtx
+	povfwctx        *parser.UnixParserCtx
+	stageFiles      []parser.Stagefile
+	povfwStageFiles []parser.Stagefile
+	nodes           []parser.Node
+	povfwnodes      []parser.Node
+	enricherCtx     []parser.EnricherCtx
+}
 
-	if err != nil {
-		return fmt.Errorf("failed to load parser config : %v", err)
+// Return new parsers
+// nodes and povfwnodes are already initialized in parser.LoadStages
+func newParsers() *parser.Parsers {
+	parsers := &parser.Parsers{
+		Ctx:             &parser.UnixParserCtx{},
+		Povfwctx:        &parser.UnixParserCtx{},
+		StageFiles:      make([]parser.Stagefile, 0),
+		PovfwStageFiles: make([]parser.Stagefile, 0),
+	}
+	for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
+		for _, hubParserItem := range cwhub.GetItemMap(itemType) {
+			if hubParserItem.Installed {
+				stagefile := parser.Stagefile{
+					Filename: hubParserItem.LocalPath,
+					Stage:    hubParserItem.Stage,
+				}
+				if itemType == cwhub.PARSERS {
+					parsers.StageFiles = append(parsers.StageFiles, stagefile)
+				}
+				if itemType == cwhub.PARSERS_OVFLW {
+					parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
+				}
+			}
+		}
 	}
-
-	log.Infof("Loading postoverflow parsers")
-	postOverflowNodes, err = parser.LoadStageDir(cConfig.ConfigFolder+"/postoverflows/", postOverflowCTX)
-
-	if err != nil {
-		return fmt.Errorf("failed to load postoverflow config : %v", err)
+	if parsers.StageFiles != nil {
+		sort.Slice(parsers.StageFiles, func(i, j int) bool {
+			return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
+		})
 	}
-
-	if cConfig.Profiling {
-		parserCTX.Profiling = true
-		postOverflowCTX.Profiling = true
+	if parsers.PovfwStageFiles != nil {
+		sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
+			return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
+		})
 	}
 
-	return nil
+	return parsers
 }
 
-func GetEnabledScenarios() string {
-	/*keep track of scenarios name for consensus profiling*/
-	var scenariosEnabled string
-	for _, x := range holders {
-		if scenariosEnabled != "" {
-			scenariosEnabled += ","
+func LoadBuckets(cConfig *csconfig.GlobalConfig) error {
+
+	var (
+		err   error
+		files []string
+	)
+	for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
+		if hubScenarioItem.Installed {
+			files = append(files, hubScenarioItem.LocalPath)
 		}
-		scenariosEnabled += x.Name
 	}
-	return scenariosEnabled
-}
 
-func LoadBuckets(cConfig *csconfig.CrowdSec) error {
-
-	var err error
-
-	log.Infof("Loading scenarios")
-	holders, outputEventChan, err = leaky.Init(map[string]string{"patterns": cConfig.ConfigFolder + "/scenarios/", "data": cConfig.DataFolder})
+	log.Infof("Loading %d scenario files", len(files))
+	holders, outputEventChan, err = leaky.LoadBuckets(cConfig.Crowdsec, files)
 
 	if err != nil {
 		return fmt.Errorf("Scenario loading failed : %v", err)
@@ -128,13 +135,13 @@ func LoadBuckets(cConfig *csconfig.CrowdSec) error {
 	buckets = leaky.NewBuckets()
 
 	/*restore as well previous state if present*/
-	if cConfig.RestoreMode != "" {
-		log.Warningf("Restoring buckets state from %s", cConfig.RestoreMode)
-		if err := leaky.LoadBucketsState(cConfig.RestoreMode, buckets, holders); err != nil {
+	if cConfig.Crowdsec.BucketStateFile != "" {
+		log.Warningf("Restoring buckets state from %s", cConfig.Crowdsec.BucketStateFile)
+		if err := leaky.LoadBucketsState(cConfig.Crowdsec.BucketStateFile, buckets, holders); err != nil {
 			return fmt.Errorf("unable to restore buckets : %s", err)
 		}
 	}
-	if cConfig.Profiling {
+	if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
 		for holderIndex := range holders {
 			holders[holderIndex].Profiling = true
 		}
@@ -142,98 +149,127 @@ func LoadBuckets(cConfig *csconfig.CrowdSec) error {
 	return nil
 }
 
-func LoadOutputs(cConfig *csconfig.CrowdSec) error {
+func LoadAcquisition(cConfig *csconfig.GlobalConfig) error {
 	var err error
-	/*
-		Load output profiles
-	*/
-	log.Infof("Loading output profiles")
-	outputProfiles, err = outputs.LoadOutputProfiles(cConfig.ConfigFolder + "/profiles.yaml")
-	if err != nil || len(outputProfiles) == 0 {
-		return fmt.Errorf("Failed to load output profiles : %v", err)
-	}
 
-	//If the user is providing a single file (ie forensic mode), don't flush expired records
-	if cConfig.SingleFile != "" {
-		log.Infof("forensic mode, disable flush")
-		cConfig.OutputConfig.Flush = false
-	} else {
-		cConfig.OutputConfig.Flush = true
-	}
-	OutputRunner, err = outputs.NewOutput(cConfig.OutputConfig)
-	if err != nil {
-		return fmt.Errorf("output plugins initialization error : %s", err.Error())
-	}
+	if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
 
-	if err := OutputRunner.StartAutoCommit(); err != nil {
-		return errors.Wrap(err, "failed to start autocommit")
-	}
+		tmpCfg := acquisition.DataSourceCfg{}
+		tmpCfg.Mode = acquisition.CAT_MODE
+		tmpCfg.Labels = map[string]string{"type": flags.SingleFileType}
+
+		if flags.SingleFilePath != "" {
+			tmpCfg.Filename = flags.SingleFilePath
+		} else if flags.SingleJournalctlFilter != "" {
+			tmpCfg.JournalctlFilters = strings.Split(flags.SingleJournalctlFilter, " ")
+		}
 
-	/* Init the API connector */
-	if cConfig.APIMode {
-		log.Infof("Loading API client")
-		var apiConfig = map[string]string{
-			"path":    cConfig.ConfigFolder + "/api.yaml",
-			"profile": GetEnabledScenarios(),
+		datasrc, err := acquisition.DataSourceConfigure(tmpCfg)
+		if err != nil {
+			return fmt.Errorf("while configuring specified file datasource : %s", err)
+		}
+		if dataSources == nil {
+			dataSources = make([]acquisition.DataSource, 0)
 		}
-		if err := OutputRunner.InitAPI(apiConfig); err != nil {
-			return fmt.Errorf("failed to load api : %s", err)
+		dataSources = append(dataSources, datasrc)
+	} else {
+		dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
+		if err != nil {
+			log.Fatalf("While loading acquisition configuration : %s", err)
 		}
 	}
+
 	return nil
 }
 
-func LoadAcquisition(cConfig *csconfig.CrowdSec) error {
-	var err error
-	//Init the acqusition : from cli or from acquis.yaml file
-	acquisitionCTX, err = acquisition.LoadAcquisitionConfig(cConfig)
-	if err != nil {
-		return fmt.Errorf("Failed to start acquisition : %s", err)
-	}
-	return nil
+func (f *Flags) Parse() {
+
+	flag.StringVar(&f.ConfigFile, "c", "/etc/crowdsec/config.yaml", "configuration file")
+	flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose")
+	flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stdout")
+	flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stdout")
+	flag.BoolVar(&f.PrintVersion, "version", false, "display version")
+	flag.StringVar(&f.SingleFilePath, "file", "", "Process a single file in time-machine")
+	flag.StringVar(&f.SingleJournalctlFilter, "jfilter", "", "Process a single journalctl output in time-machine")
+	flag.StringVar(&f.SingleFileType, "type", "", "Labels.type for file in time-machine")
+	flag.BoolVar(&f.TestMode, "t", false, "only test configs")
+	flag.BoolVar(&f.DisableAgent, "no-cs", false, "disable crowdsec agent")
+	flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
+
+	flag.Parse()
 }
 
-func StartProcessingRoutines(cConfig *csconfig.CrowdSec) (chan types.Event, error) {
+// LoadConfig return configuration parsed from configuration file
+func LoadConfig(config *csconfig.GlobalConfig) error {
 
-	acquisTomb = tomb.Tomb{}
-	parsersTomb = tomb.Tomb{}
-	bucketsTomb = tomb.Tomb{}
-	outputsTomb = tomb.Tomb{}
+	if flags.ConfigFile != "" {
+		if err := config.LoadConfigurationFile(flags.ConfigFile); err != nil {
+			return fmt.Errorf("while loading configuration : %s", err)
+		}
+	} else {
+		log.Warningf("no configuration file provided")
+	}
+	disableAPI = flags.DisableAPI
+	disableAgent = flags.DisableAgent
 
-	inputLineChan := make(chan types.Event)
-	inputEventChan := make(chan types.Event)
+	if !disableAPI && (cConfig.API == nil || cConfig.API.Server == nil) {
+		log.Errorf("no API server configuration found, will not start the local API")
+		disableAPI = true
+	}
 
-	//start go-routines for parsing, buckets pour and ouputs.
-	for i := 0; i < cConfig.NbParsers; i++ {
-		parsersTomb.Go(func() error {
-			err := runParse(inputLineChan, inputEventChan, *parserCTX, parserNodes)
-			if err != nil {
-				log.Errorf("runParse error : %s", err)
-				return err
-			}
-			return nil
-		})
+	if !disableAgent && cConfig.Crowdsec == nil {
+		log.Errorf("no configuration found crowdsec agent, will not start the agent")
+		disableAgent = true
 	}
 
-	bucketsTomb.Go(func() error {
-		err := runPour(inputEventChan, holders, buckets)
-		if err != nil {
-			log.Errorf("runPour error : %s", err)
-			return err
+	if !disableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
+		log.Fatalf("missing local API credentials for crowdsec agent, abort")
+	}
+
+	if disableAPI && disableAgent {
+		log.Fatalf("You must run at least the API Server or crowdsec")
+	}
+
+	if flags.SingleFilePath != "" {
+		if flags.SingleFileType == "" {
+			return fmt.Errorf("-file requires -type")
+		}
+	}
+
+	if flags.SingleJournalctlFilter != "" {
+		if flags.SingleFileType == "" {
+			return fmt.Errorf("-jfilter requires -type")
 		}
-		return nil
-	})
+	}
 
-	outputsTomb.Go(func() error {
-		err := runOutput(inputEventChan, outputEventChan, holders, buckets, *postOverflowCTX, postOverflowNodes, outputProfiles, OutputRunner)
-		if err != nil {
-			log.Errorf("runPour error : %s", err)
-			return err
+	if flags.DebugLevel {
+		logLevel := log.DebugLevel
+		config.Common.LogLevel = &logLevel
+	}
+	if flags.InfoLevel || config.Common.LogLevel == nil {
+		logLevel := log.InfoLevel
+		config.Common.LogLevel = &logLevel
+	}
+	if flags.TraceLevel {
+		logLevel := log.TraceLevel
+		config.Common.LogLevel = &logLevel
+	}
+
+	if flags.TestMode && !disableAgent {
+		config.Crowdsec.LintOnly = true
+	}
+
+	if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
+		config.API.Server.OnlineClient = nil
+		/*if the api is disabled as well, just read file and exit, don't daemonize*/
+		if disableAPI {
+			config.Common.Daemonize = false
 		}
-		return nil
-	})
+		config.Common.LogMedia = "stdout"
+		log.Infof("single file mode : log_media=%s daemonize=%t", config.Common.LogMedia, config.Common.Daemonize)
+	}
 
-	return inputLineChan, nil
+	return nil
 }
 
 func main() {
@@ -241,114 +277,47 @@ func main() {
 		err error
 	)
 
-	cConfig = csconfig.NewCrowdSecConfig()
+	defer types.CatchPanic("crowdsec/main")
 
+	cConfig = csconfig.NewConfig()
 	// Handle command line arguments
-	if err := cConfig.LoadConfig(); err != nil {
+	flags = &Flags{}
+	flags.Parse()
+	if flags.PrintVersion {
+		cwversion.Show()
+		os.Exit(0)
+	}
+
+	if err := LoadConfig(cConfig); err != nil {
 		log.Fatalf(err.Error())
 	}
 	// Configure logging
-	if err = types.SetDefaultLoggerConfig(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil {
+	if err = types.SetDefaultLoggerConfig(cConfig.Common.LogMedia, cConfig.Common.LogDir, *cConfig.Common.LogLevel); err != nil {
 		log.Fatal(err.Error())
 	}
 
-	daemonCTX := &daemon.Context{
-		PidFileName: cConfig.PIDFolder + "/crowdsec.pid",
-		PidFilePerm: 0644,
-		WorkDir:     "./",
-		Umask:       027,
-	}
-	if cConfig.Daemonize {
-		daemon.SetSigHandler(termHandler, syscall.SIGTERM)
-		daemon.SetSigHandler(reloadHandler, syscall.SIGHUP)
-		daemon.SetSigHandler(debugHandler, syscall.SIGUSR1)
-
-		d, err := daemonCTX.Reborn()
-		if err != nil {
-			log.Fatalf("unable to run daemon: %s ", err.Error())
-		}
-		if d != nil {
-			return
-		}
-	}
-
 	log.Infof("Crowdsec %s", cwversion.VersionStr())
 
-	// Enable profiling early
-	if cConfig.Prometheus {
-		registerPrometheus(cConfig.PrometheusMode)
-		cConfig.Profiling = true
-	}
-	if cConfig.Profiling {
-		go runTachymeter(cConfig.HTTPListen)
-	}
-
-	err = exprhelpers.Init()
-	if err != nil {
-		log.Fatalf("Failed to init expr helpers : %s", err)
-	}
-
-	// Start loading configs
-	if err := LoadParsers(cConfig); err != nil {
-		log.Fatalf("Failed to load parsers: %s", err)
+	if !flags.DisableAPI && (cConfig.API == nil || cConfig.API.Server == nil) {
+		log.Errorf("no API server configuration found, will not start the local API")
+		flags.DisableAPI = true
 	}
 
-	if err := LoadBuckets(cConfig); err != nil {
-		log.Fatalf("Failed to load scenarios: %s", err)
+	if !flags.DisableAgent && cConfig.Crowdsec == nil {
+		log.Errorf("no configuration found crowdsec agent, will not start the agent")
+		flags.DisableAgent = true
 	}
 
-	if err := LoadOutputs(cConfig); err != nil {
-		log.Fatalf("failed to initialize outputs : %s", err)
+	if !flags.DisableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
+		log.Fatalf("missing local API credentials for crowdsec agent, abort")
 	}
-
-	if err := LoadAcquisition(cConfig); err != nil {
-		log.Fatalf("Error while loading acquisition config : %s", err)
-	}
-
-	/* if it's just linting, we're done */
-	if cConfig.Linter {
-		return
-	}
-
-	/*if the user is in "single file mode" (might be writting scenario or parsers),
-	allow loading **without** parsers or scenarios */
-	if cConfig.SingleFile == "" {
-		if len(parserNodes) == 0 {
-			log.Fatalf("no parser(s) loaded, abort.")
-		}
-
-		if len(holders) == 0 {
-			log.Fatalf("no bucket(s) loaded, abort.")
-		}
-
-		if len(outputProfiles) == 0 {
-			log.Fatalf("no output profile(s) loaded, abort.")
-		}
+	// Enable profiling early
+	if cConfig.Prometheus != nil {
+		go registerPrometheus(cConfig.Prometheus)
 	}
 
-	//Start the background routines that comunicate via chan
-	log.Infof("Starting processing routines")
-	inputLineChan, err := StartProcessingRoutines(cConfig)
-	if err != nil {
-		log.Fatalf("failed to start processing routines : %s", err)
+	if err := Serve(); err != nil {
+		log.Fatalf(err.Error())
 	}
 
-	//Fire!
-	log.Warningf("Starting processing data")
-
-	acquisition.AcquisStartReading(acquisitionCTX, inputLineChan, &acquisTomb)
-
-	if !cConfig.Daemonize {
-		if err = serveOneTimeRun(*OutputRunner); err != nil {
-			log.Errorf(err.Error())
-		} else {
-			return
-		}
-	} else {
-		defer daemonCTX.Release() //nolint:errcheck // won't bother checking this error in defer statement
-		err = daemon.ServeSignals()
-		if err != nil {
-			log.Fatalf("serveDaemon error : %s", err.Error())
-		}
-	}
 }

cmd/crowdsec/metrics.go

@@ -1,33 +1,22 @@
 package main
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+	v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
-	"github.com/jamiealquiza/tachymeter"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"net/http"
 
 	log "github.com/sirupsen/logrus"
-
-	"runtime"
-)
-
-var (
-	parseStat     *tachymeter.Tachymeter
-	bucketStat    *tachymeter.Tachymeter
-	outputStat    *tachymeter.Tachymeter
-	linesReadOK   uint64
-	linesReadKO   uint64
-	linesParsedOK uint64
-	linesParsedKO uint64
-	linesPouredOK uint64
-	linesPouredKO uint64
 )
 
 /*prometheus*/
@@ -79,69 +68,49 @@ func dumpMetrics() {
 	var tmpFile string
 	var err error
 
-	if cConfig.DumpBuckets {
+	if cConfig.Crowdsec.BucketStateDumpDir != "" {
 		log.Infof("!! Dumping buckets state")
-		if tmpFile, err = leaky.DumpBucketsStateAt(time.Now(), buckets); err != nil {
+		if tmpFile, err = leaky.DumpBucketsStateAt(time.Now(), cConfig.Crowdsec.BucketStateDumpDir, buckets); err != nil {
 			log.Fatalf("Failed dumping bucket state : %s", err)
 		}
 		log.Infof("Buckets state dumped to %s", tmpFile)
 	}
-
-	if cConfig.Profiling {
-		var memoryStats runtime.MemStats
-		runtime.ReadMemStats(&memoryStats)
-
-		log.Infof("parser evt/s : %s", parseStat.Calc())
-		log.Infof("bucket pour evt/s : %s", bucketStat.Calc())
-		log.Infof("outputs evt/s : %s", outputStat.Calc())
-		log.Infof("Alloc = %v MiB", bToMb(memoryStats.Alloc))
-		log.Infof("TotalAlloc = %v MiB", bToMb(memoryStats.TotalAlloc))
-		log.Infof("Sys = %v MiB", bToMb(memoryStats.Sys))
-		log.Infof("NumGC = %v", memoryStats.NumGC)
-		log.Infof("Lines read ok : %d", linesReadOK)
-		if linesReadKO > 0 {
-			log.Infof("Lines discarded : %d (%.2f%%)", linesReadKO, float64(linesReadKO)/float64(linesReadOK)*100.0)
-		}
-		log.Infof("Lines parsed ok : %d", linesParsedOK)
-		if linesParsedKO > 0 {
-			log.Infof("Lines unparsed : %d (%.2f%%)", linesParsedKO, float64(linesParsedKO)/float64(linesParsedOK)*100.0)
-		}
-		log.Infof("Lines poured ok : %d", linesPouredOK)
-		if linesPouredKO > 0 {
-			log.Infof("Lines never poured : %d (%.2f%%)", linesPouredKO, float64(linesPouredKO)/float64(linesPouredOK)*100.0)
-		}
-		log.Infof("Writting metrics dump to %s", cConfig.WorkingFolder+"/crowdsec.profile")
-		if err := prometheus.WriteToTextfile(cConfig.WorkingFolder+"/crowdsec.profile", prometheus.DefaultGatherer); err != nil {
-			log.Errorf("failed to write metrics to %s : %s", cConfig.WorkingFolder+"/crowdsec.profile", err)
-		}
-	}
 }
 
-func runTachymeter(HTTPListen string) {
-	log.Warningf("Starting profiling and http server")
-	/*Tachymeter for global perfs */
-	parseStat = tachymeter.New(&tachymeter.Config{Size: 100})
-	bucketStat = tachymeter.New(&tachymeter.Config{Size: 100})
-	outputStat = tachymeter.New(&tachymeter.Config{Size: 100})
-	log.Fatal(http.ListenAndServe(HTTPListen, nil))
-}
+func registerPrometheus(config *csconfig.PrometheusCfg) {
+	if !config.Enabled {
+		return
+	}
+	if config.ListenAddr == "" {
+		log.Warningf("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
+		config.ListenAddr = "127.0.0.1"
+	}
+	if config.ListenPort == 0 {
+		log.Warningf("prometheus is enabled, but the listen port is empty, using '6060'")
+		config.ListenPort = 6060
+	}
 
-func registerPrometheus(mode string) {
+	defer types.CatchPanic("crowdsec/registerPrometheus")
 	/*Registering prometheus*/
 	/*If in aggregated mode, do not register events associated to a source, keeps cardinality low*/
-	if mode == "aggregated" {
+	if config.Level == "aggregated" {
 		log.Infof("Loading aggregated prometheus collectors")
 		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
 			acquisition.ReaderHits, globalCsInfo,
 			leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow,
+			v1.LapiRouteHits,
 			leaky.BucketsCurrentCount)
 	} else {
 		log.Infof("Loading prometheus collectors")
 		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
 			parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo,
 			acquisition.ReaderHits, globalCsInfo,
+			v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions,
 			leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsInstanciation, leaky.BucketsOverflow, leaky.BucketsCurrentCount)
 
 	}
 	http.Handle("/metrics", promhttp.Handler())
+	if err := http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenAddr, config.ListenPort), nil); err != nil {
+		log.Warningf("prometheus: %s", err)
+	}
 }

cmd/crowdsec/output.go

@@ -1,79 +1,158 @@
 package main
 
 import (
+	"context"
 	"fmt"
-
-	log "github.com/sirupsen/logrus"
-
+	"net/url"
+	"sync"
 	"time"
 
+	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
-	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 )
 
-func runOutput(input chan types.Event, overflow chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets,
-	poctx parser.UnixParserCtx, ponodes []parser.Node, outputProfiles []types.Profile, output *outputs.Output) error {
-	var (
-		//action string
-		start time.Time
-	)
+func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
+
+	var dedupCache []*models.Alert
+
+	for idx, alert := range alerts {
+		log.Tracef("alert %d/%d", idx, len(alerts))
+		/*if we have more than one source, we need to dedup */
+		if len(alert.Sources) == 0 || len(alert.Sources) == 1 {
+			dedupCache = append(dedupCache, alert.Alert)
+			continue
+		}
+		for k, src := range alert.Sources {
+			refsrc := *alert.Alert //copy
+			log.Tracef("source[%s]", k)
+			refsrc.Source = &src
+			dedupCache = append(dedupCache, &refsrc)
+		}
+	}
+	if len(dedupCache) != len(alerts) {
+		log.Tracef("went from %d to %d alerts", len(alerts), len(dedupCache))
+	}
+	return dedupCache, nil
+}
+
+func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error {
+	ctx := context.Background()
+	alertsToPush, err := dedupAlerts(alerts)
+
+	if err != nil {
+		return errors.Wrap(err, "failed to transform alerts for api")
+	}
+	_, _, err = client.Alerts.Add(ctx, alertsToPush)
+	if err != nil {
+		return errors.Wrap(err, "failed sending alert to LAPI")
+	}
+	return nil
+}
+
+func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
+	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
+
+	var err error
+	ticker := time.NewTicker(1 * time.Second)
+
+	var cache []types.RuntimeAlert
+	var cacheMutex sync.Mutex
 
+	scenarios, err := cwhub.GetUpstreamInstalledScenariosAsString()
+	if err != nil {
+		return errors.Wrapf(err, "loading list of installed hub scenarios: %s", err)
+	}
+
+	apiURL, err := url.Parse(apiConfig.URL)
+	if err != nil {
+		return errors.Wrapf(err, "parsing api url ('%s'): %s", apiConfig.URL, err)
+	}
+
+	password := strfmt.Password(apiConfig.Password)
+
+	Client, err := apiclient.NewClient(&apiclient.Config{
+		MachineID:      apiConfig.Login,
+		Password:       password,
+		Scenarios:      scenarios,
+		UserAgent:      fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
+		URL:            apiURL,
+		VersionPrefix:  "v1",
+		UpdateScenario: cwhub.GetUpstreamInstalledScenariosAsString,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "new client api: %s", err)
+	}
+	if _, err = Client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
+		MachineID: &apiConfig.Login,
+		Password:  &password,
+		Scenarios: scenarios,
+	}); err != nil {
+		return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login)
+	}
 LOOP:
 	for {
 		select {
+		case <-ticker.C:
+			if len(cache) > 0 {
+				cacheMutex.Lock()
+				cachecopy := cache
+				newcache := make([]types.RuntimeAlert, 0)
+				cache = newcache
+				cacheMutex.Unlock()
+				if err := PushAlerts(cachecopy, Client); err != nil {
+					log.Errorf("while pushing to api : %s", err)
+				}
+			}
 		case <-outputsTomb.Dying():
-			log.Infof("Flushing outputs")
-			output.FlushAll()
-			log.Debugf("Shuting down output routines")
-			if err := output.Shutdown(); err != nil {
-				log.Errorf("error while in output shutdown: %s", err)
+			if len(cache) > 0 {
+				cacheMutex.Lock()
+				cachecopy := cache
+				newcache := make([]types.RuntimeAlert, 0)
+				cache = newcache
+				cacheMutex.Unlock()
+				if err := PushAlerts(cachecopy, Client); err != nil {
+					log.Errorf("while pushing leftovers to api : %s", err)
+				}
 			}
-			log.Infof("Done shutdown down output")
 			break LOOP
 		case event := <-overflow:
-			//if global simulation -> everything is simulation unless told otherwise
-			if cConfig.SimulationCfg != nil && cConfig.SimulationCfg.Simulation {
-				event.Overflow.Simulation = true
-			}
-			if cConfig.Profiling {
-				start = time.Now()
-			}
 
+			/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
+			if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
+				buckets.Bucket_map.Delete(event.Overflow.Mapkey)
+				break
+			}
 			if event.Overflow.Reprocess {
 				log.Debugf("Overflow being reprocessed.")
 				input <- event
 			}
-
 			/* process post overflow parser nodes */
-			event, err := parser.Parse(poctx, event, ponodes)
+			event, err := parser.Parse(postOverflowCTX, event, postOverflowNodes)
 			if err != nil {
 				return fmt.Errorf("postoverflow failed : %s", err)
 			}
-			//check scenarios in simulation
-			if cConfig.SimulationCfg != nil {
-				for _, scenario_name := range cConfig.SimulationCfg.Exclusions {
-					if event.Overflow.Scenario == scenario_name {
-						event.Overflow.Simulation = !event.Overflow.Simulation
-					}
-				}
+			log.Printf("%s", *event.Overflow.Alert.Message)
+			if event.Overflow.Whitelisted {
+				log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
+				continue
 			}
+			cacheMutex.Lock()
+			cache = append(cache, event.Overflow)
+			cacheMutex.Unlock()
 
-			if event.Overflow.Scenario == "" && event.Overflow.MapKey != "" {
-				//log.Infof("Deleting expired entry %s", event.Overflow.MapKey)
-				buckets.Bucket_map.Delete(event.Overflow.MapKey)
-			} else {
-				/*let's handle output profiles */
-				if err := output.ProcessOutput(event.Overflow, outputProfiles); err != nil {
-					log.Warningf("Error while processing overflow/output : %s", err)
-				}
-			}
-		}
-		if cConfig.Profiling {
-			outputStat.AddTime(time.Since(start))
 		}
 	}
+
+	ticker.Stop()
 	return nil
 
 }

cmd/crowdsec/parse.go

@@ -2,8 +2,6 @@ package main
 
 import (
 	"errors"
-	"sync/atomic"
-	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
@@ -13,8 +11,6 @@ import (
 )
 
 func runParse(input chan types.Event, output chan types.Event, parserCTX parser.UnixParserCtx, nodes []parser.Node) error {
-	var start time.Time
-	var discardCPT, processCPT int
 
 LOOP:
 	for {
@@ -23,18 +19,9 @@ LOOP:
 			log.Infof("Killing parser routines")
 			break LOOP
 		case event := <-input:
-			if cConfig.Profiling {
-				start = time.Now()
-			}
 			if !event.Process {
-				if cConfig.Profiling {
-					atomic.AddUint64(&linesReadKO, 1)
-				}
 				continue
 			}
-			if cConfig.Profiling {
-				atomic.AddUint64(&linesReadOK, 1)
-			}
 			globalParserHits.With(prometheus.Labels{"source": event.Line.Src}).Inc()
 
 			/* parse the log using magic */
@@ -44,30 +31,16 @@ LOOP:
 				return errors.New("parsing failed :/")
 			}
 			if !parsed.Process {
-				if cConfig.Profiling {
-					atomic.AddUint64(&linesParsedKO, 1)
-				}
 				globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src}).Inc()
 				log.Debugf("Discarding line %+v", parsed)
-				discardCPT++
 				continue
 			}
-			if cConfig.Profiling {
-				atomic.AddUint64(&linesParsedOK, 1)
-			}
 			globalParserHitsOk.With(prometheus.Labels{"source": event.Line.Src}).Inc()
-			processCPT++
 			if parsed.Whitelisted {
 				log.Debugf("event whitelisted, discard")
 				continue
 			}
-			if processCPT%1000 == 0 {
-				log.Debugf("%d lines processed, %d lines discarded (unparsed)", processCPT, discardCPT)
-			}
 			output <- parsed
-			if cConfig.Profiling {
-				parseStat.AddTime(time.Since(start))
-			}
 		}
 	}
 	return nil

cmd/crowdsec/pour.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"sync/atomic"
 	"time"
 
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -12,34 +11,29 @@ import (
 
 func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets) error {
 	var (
-		start time.Time
 		count int
 	)
-LOOP:
 	for {
 		//bucket is now ready
 		select {
 		case <-bucketsTomb.Dying():
-			log.Infof("Exiting pour routine")
-
-			break LOOP
+			log.Infof("Bucket routine exiting")
+			return nil
 		case parsed := <-input:
 			count++
-			if cConfig.Profiling {
-				start = time.Now()
-			}
-
 			if count%5000 == 0 {
 				log.Warningf("%d existing LeakyRoutine", leaky.LeakyRoutineCount)
 				//when in forensics mode, garbage collect buckets
-				if parsed.MarshaledTime != "" && cConfig.SingleFile != "" {
-					var z *time.Time = &time.Time{}
-					if err := z.UnmarshalText([]byte(parsed.MarshaledTime)); err != nil {
-						log.Warningf("Failed to unmarshal time from event '%s' : %s", parsed.MarshaledTime, err)
-					} else {
-						log.Warningf("Starting buckets garbage collection ...")
-						if err = leaky.GarbageCollectBuckets(*z, buckets); err != nil {
-							return fmt.Errorf("failed to start bucket GC : %s", err)
+				if cConfig.Crowdsec.BucketsGCEnabled {
+					if parsed.MarshaledTime != "" {
+						var z *time.Time = &time.Time{}
+						if err := z.UnmarshalText([]byte(parsed.MarshaledTime)); err != nil {
+							log.Warningf("Failed to unmarshal time from event '%s' : %s", parsed.MarshaledTime, err)
+						} else {
+							log.Warningf("Starting buckets garbage collection ...")
+							if err = leaky.GarbageCollectBuckets(*z, buckets); err != nil {
+								return fmt.Errorf("failed to start bucket GC : %s", err)
+							}
 						}
 					}
 				}
@@ -52,22 +46,14 @@ LOOP:
 			}
 			if poured {
 				globalBucketPourOk.Inc()
-				atomic.AddUint64(&linesPouredOK, 1)
 			} else {
 				globalBucketPourKo.Inc()
-				atomic.AddUint64(&linesPouredKO, 1)
-			}
-			if cConfig.Profiling {
-				bucketStat.AddTime(time.Since(start))
 			}
 			if len(parsed.MarshaledTime) != 0 {
 				if err := lastProcessedItem.UnmarshalText([]byte(parsed.MarshaledTime)); err != nil {
-					log.Debugf("failed to unmarshal time from event : %s", err)
+					log.Warningf("failed to unmarshal time from event : %s", err)
 				}
 			}
-
 		}
 	}
-	log.Infof("Sending signal Bucketify")
-	return nil
 }

cmd/crowdsec/serve.go

@@ -1,15 +1,20 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
-	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+	"github.com/coreos/go-systemd/daemon"
+	"github.com/pkg/errors"
+
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
-	"github.com/crowdsecurity/crowdsec/pkg/outputs"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 	log "github.com/sirupsen/logrus"
-
-	"github.com/sevlyar/go-daemon"
+	"gopkg.in/tomb.v2"
+	//"github.com/sevlyar/go-daemon"
 )
 
 //debugHandler is kept as a dev convenience : it shuts down and serialize internal state
@@ -17,11 +22,11 @@ func debugHandler(sig os.Signal) error {
 	var tmpFile string
 	var err error
 	//stop go routines
-	if err := ShutdownRoutines(); err != nil {
+	if err := ShutdownCrowdsecRoutines(); err != nil {
 		log.Warningf("Failed to shut down routines: %s", err)
 	}
 	//todo : properly stop acquis with the tail readers
-	if tmpFile, err = leaky.DumpBucketsStateAt(time.Now(), buckets); err != nil {
+	if tmpFile, err = leaky.DumpBucketsStateAt(time.Now(), cConfig.Crowdsec.BucketStateDumpDir, buckets); err != nil {
 		log.Warningf("Failed dumping bucket state : %s", err)
 	}
 	if err := leaky.ShutdownAllBuckets(buckets); err != nil {
@@ -34,154 +39,237 @@ func debugHandler(sig os.Signal) error {
 func reloadHandler(sig os.Signal) error {
 	var tmpFile string
 	var err error
+
 	//stop go routines
-	if err := ShutdownRoutines(); err != nil {
-		log.Fatalf("Failed to shut down routines: %s", err)
-	}
-	if tmpFile, err = leaky.DumpBucketsStateAt(time.Now(), buckets); err != nil {
-		log.Fatalf("Failed dumping bucket state : %s", err)
-	}
+	if !disableAgent {
+		if err := shutdownCrowdsec(); err != nil {
+			log.Fatalf("Failed to shut down crowdsec routines: %s", err)
+		}
+		if cConfig.Crowdsec != nil && cConfig.Crowdsec.BucketStateDumpDir != "" {
+			if tmpFile, err = leaky.DumpBucketsStateAt(time.Now(), cConfig.Crowdsec.BucketStateDumpDir, buckets); err != nil {
+				log.Fatalf("Failed dumping bucket state : %s", err)
+			}
+		}
 
-	if err := leaky.ShutdownAllBuckets(buckets); err != nil {
-		log.Fatalf("while shutting down routines : %s", err)
-	}
-	//reload the simulation state
-	if err := cConfig.LoadSimulation(); err != nil {
-		log.Errorf("reload error (simulation) : %s", err)
+		if err := leaky.ShutdownAllBuckets(buckets); err != nil {
+			log.Fatalf("while shutting down routines : %s", err)
+		}
 	}
 
-	//reload all and start processing again :)
-	if err := LoadParsers(cConfig); err != nil {
-		log.Fatalf("Failed to load parsers: %s", err)
+	if !disableAPI {
+		if err := shutdownAPI(); err != nil {
+			log.Fatalf("Failed to shut down api routines: %s", err)
+		}
 	}
 
-	if err := LoadBuckets(cConfig); err != nil {
-		log.Fatalf("Failed to load scenarios: %s", err)
+	/*
+	 re-init tombs
+	*/
+	acquisTomb = tomb.Tomb{}
+	parsersTomb = tomb.Tomb{}
+	bucketsTomb = tomb.Tomb{}
+	outputsTomb = tomb.Tomb{}
+	apiTomb = tomb.Tomb{}
+	crowdsecTomb = tomb.Tomb{}
 
+	if err := LoadConfig(cConfig); err != nil {
+		log.Fatalf(err.Error())
 	}
-	//restore bucket state
-	log.Warningf("Restoring buckets state from %s", tmpFile)
-	if err := leaky.LoadBucketsState(tmpFile, buckets, holders); err != nil {
-		log.Fatalf("unable to restore buckets : %s", err)
+	// Configure logging
+	if err = types.SetDefaultLoggerConfig(cConfig.Common.LogMedia, cConfig.Common.LogDir, *cConfig.Common.LogLevel); err != nil {
+		log.Fatal(err.Error())
 	}
 
-	if err := LoadOutputs(cConfig); err != nil {
-		log.Fatalf("failed to initialize outputs : %s", err)
-	}
+	if !disableAPI {
+		apiServer, err := initAPIServer()
+		if err != nil {
+			return fmt.Errorf("unable to init api server: %s", err)
+		}
 
-	if err := LoadAcquisition(cConfig); err != nil {
-		log.Fatalf("Error while loading acquisition config : %s", err)
-	}
-	//Start the background routines that comunicate via chan
-	log.Infof("Starting processing routines")
-	inputLineChan, err := StartProcessingRoutines(cConfig)
-	if err != nil {
-		log.Fatalf("failed to start processing routines : %s", err)
+		serveAPIServer(apiServer)
 	}
 
-	//Fire!
-	log.Warningf("Starting processing data")
-
-	acquisition.AcquisStartReading(acquisitionCTX, inputLineChan, &acquisTomb)
+	if !disableAgent {
+		csParsers, err := initCrowdsec()
+		if err != nil {
+			return fmt.Errorf("unable to init crowdsec: %s", err)
+		}
+		//restore bucket state
+		if tmpFile != "" {
+			log.Warningf("Restoring buckets state from %s", tmpFile)
+			if err := leaky.LoadBucketsState(tmpFile, buckets, holders); err != nil {
+				log.Fatalf("unable to restore buckets : %s", err)
+			}
+		}
+		//reload the simulation state
+		if err := cConfig.LoadSimulation(); err != nil {
+			log.Errorf("reload error (simulation) : %s", err)
+		}
+		serveCrowdsec(csParsers)
+	}
 
 	log.Printf("Reload is finished")
 	//delete the tmp file, it's safe now :)
-	if err := os.Remove(tmpFile); err != nil {
-		log.Warningf("Failed to delete temp file (%s) : %s", tmpFile, err)
+	if tmpFile != "" {
+		if err := os.Remove(tmpFile); err != nil {
+			log.Warningf("Failed to delete temp file (%s) : %s", tmpFile, err)
+		}
 	}
 	return nil
 }
 
-func ShutdownRoutines() error {
+func ShutdownCrowdsecRoutines() error {
 	var reterr error
 
+	log.Debugf("Shutting down crowdsec sub-routines")
 	acquisTomb.Kill(nil)
-	log.Infof("waiting for acquisition to finish")
+	log.Debugf("waiting for acquisition to finish")
 	if err := acquisTomb.Wait(); err != nil {
 		log.Warningf("Acquisition returned error : %s", err)
 		reterr = err
 	}
-	log.Infof("acquisition is finished, wait for parser/bucket/ouputs.")
+	log.Debugf("acquisition is finished, wait for parser/bucket/ouputs.")
 	parsersTomb.Kill(nil)
 	if err := parsersTomb.Wait(); err != nil {
 		log.Warningf("Parsers returned error : %s", err)
 		reterr = err
 	}
-	log.Infof("parsers is done")
+	log.Debugf("parsers is done")
 	bucketsTomb.Kill(nil)
 	if err := bucketsTomb.Wait(); err != nil {
 		log.Warningf("Buckets returned error : %s", err)
 		reterr = err
 	}
-	log.Infof("buckets is done")
+	log.Debugf("buckets is done")
 	outputsTomb.Kill(nil)
 	if err := outputsTomb.Wait(); err != nil {
 		log.Warningf("Ouputs returned error : %s", err)
 		reterr = err
 
 	}
-	log.Infof("outputs are done")
+	log.Debugf("outputs are done")
+	//everything is dead johny
+	crowdsecTomb.Kill(nil)
+
 	return reterr
 }
 
-func termHandler(sig os.Signal) error {
-	log.Infof("Shutting down routines")
-	if err := ShutdownRoutines(); err != nil {
-		log.Errorf("Error encountered while shutting down routines : %s", err)
+func shutdownAPI() error {
+	log.Debugf("shutting down api via Tomb")
+	apiTomb.Kill(nil)
+	if err := apiTomb.Wait(); err != nil {
+		return err
 	}
-	log.Warningf("all routines are done, bye.")
-	return daemon.ErrStop
+	log.Debugf("done")
+	return nil
 }
 
-func serveOneTimeRun(outputRunner outputs.Output) error {
-	if err := acquisTomb.Wait(); err != nil {
-		log.Warningf("acquisition returned error : %s", err)
+func shutdownCrowdsec() error {
+	log.Debugf("shutting down crowdsec via Tomb")
+	crowdsecTomb.Kill(nil)
+	if err := crowdsecTomb.Wait(); err != nil {
+		return err
 	}
-	log.Infof("acquisition is finished, wait for parser/bucket/ouputs.")
-
-	/*
-		While it might make sense to want to shut-down parser/buckets/etc. as soon as acquisition is finished,
-		we might have some pending buckets : buckets that overflowed, but which LeakRoutine are still alive because they
-		are waiting to be able to "commit" (push to db). This can happens specifically in a context where a lot of logs
-		are going to trigger overflow (ie. trigger buckets with ~100% of the logs triggering an overflow).
+	log.Debugf("done")
+	return nil
+}
 
-		To avoid this (which would mean that we would "lose" some overflows), let's monitor the number of live buckets.
-		However, because of the blackhole mechanism, you can't really wait for the number of LeakRoutine to go to zero (we might have to wait $blackhole_duration).
+func termHandler(sig os.Signal) error {
+	if err := shutdownCrowdsec(); err != nil {
+		log.Errorf("Error encountered while shutting down crowdsec: %s", err)
+	}
+	if err := shutdownAPI(); err != nil {
+		log.Errorf("Error encountered while shutting down api: %s", err)
+	}
+	log.Debugf("termHandler done")
+	return nil
+}
 
-		So : we are waiting for the number of buckets to stop decreasing before returning. "how long" we should wait is a bit of the trick question,
-		as some operations (ie. reverse dns or such in post-overflow) can take some time :)
-	*/
+func HandleSignals() {
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan,
+		syscall.SIGHUP,
+		syscall.SIGTERM)
 
-	bucketCount := leaky.LeakyRoutineCount
-	rounds := 0
-	successiveStillRounds := 0
-	for {
-		rounds++
-		time.Sleep(5 * time.Second)
-		currBucketCount := leaky.LeakyRoutineCount
-		if currBucketCount != bucketCount {
-			if rounds == 0 || rounds%2 == 0 {
-				log.Printf("Still %d live LeakRoutines, waiting (was %d)", currBucketCount, bucketCount)
-			}
-			bucketCount = currBucketCount
-			successiveStillRounds = 0
-		} else {
-			if successiveStillRounds > 1 {
-				log.Printf("LeakRoutines commit over.")
-				break
+	exitChan := make(chan int)
+	go func() {
+		defer types.CatchPanic("crowdsec/HandleSignals")
+		for {
+			s := <-signalChan
+			switch s {
+			// kill -SIGHUP XXXX
+			case syscall.SIGHUP:
+				log.Warningf("SIGHUP received, reloading")
+				if err := reloadHandler(s); err != nil {
+					log.Fatalf("Reload handler failure : %s", err)
+				}
+			// kill -SIGTERM XXXX
+			case syscall.SIGTERM:
+				log.Warningf("SIGTERM received, shutting down")
+				if err := termHandler(s); err != nil {
+					log.Fatalf("Term handler failure : %s", err)
+				}
+				exitChan <- 0
 			}
-			successiveStillRounds++
+		}
+	}()
+
+	code := <-exitChan
+	log.Warningf("Crowdsec service shutting down")
+	os.Exit(code)
+}
+
+func Serve() error {
+	acquisTomb = tomb.Tomb{}
+	parsersTomb = tomb.Tomb{}
+	bucketsTomb = tomb.Tomb{}
+	outputsTomb = tomb.Tomb{}
+	apiTomb = tomb.Tomb{}
+	crowdsecTomb = tomb.Tomb{}
+
+	if !disableAPI {
+		apiServer, err := initAPIServer()
+		if err != nil {
+			return errors.Wrap(err, "api server init")
+		}
+		if !flags.TestMode {
+			serveAPIServer(apiServer)
 		}
 	}
 
-	time.Sleep(5 * time.Second)
+	if !disableAgent {
+		csParsers, err := initCrowdsec()
+		if err != nil {
+			return errors.Wrap(err, "crowdsec init")
+		}
+		/* if it's just linting, we're done */
+		if !flags.TestMode {
+			serveCrowdsec(csParsers)
+		}
+	}
+	if flags.TestMode {
+		log.Infof("test done")
+		os.Exit(0)
+	}
 
-	// wait for the parser to parse all events
-	if err := ShutdownRoutines(); err != nil {
-		log.Errorf("failed shutting down routines : %s", err)
+	if cConfig.Common != nil && cConfig.Common.Daemonize {
+		sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)
+		if !sent || err != nil {
+			log.Errorf("Failed to notify(sent: %v): %v", sent, err)
+		}
+		/*wait for signals*/
+		HandleSignals()
+	} else {
+		for {
+			select {
+			case <-apiTomb.Dead():
+				log.Infof("api shutdown")
+				os.Exit(0)
+			case <-crowdsecTomb.Dead():
+				log.Infof("crowdsec shutdown")
+				os.Exit(0)
+			}
+		}
 	}
-	dumpMetrics()
-	outputRunner.Flush()
-	log.Warningf("all routines are done, bye.")
 	return nil
 }

config/api.yaml

@@ -1,8 +0,0 @@
-version: v1
-url: https://tmsov6x2n9.execute-api.eu-west-1.amazonaws.com
-signin_path: signin
-push_path: signals
-pull_path: pull
-enroll_path: enroll
-reset_pwd_path: resetpassword
-register_path: register

config/config.yaml

@@ -0,0 +1,49 @@
+common:
+  daemonize: true
+  pid_dir: /var/run/
+  log_media: file
+  log_level: info
+  log_dir: /var/log/
+  working_dir: .
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+  hub_branch: wip_lapi
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  #user: 
+  #password:
+  #db_name:
+  #host:
+  #port:
+  flush:
+    max_items: 5000
+    max_age: 7d
+api:
+  client:
+    insecure_skip_verify: true
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    log_level: info
+    listen_uri: localhost:8080
+    profiles_path: /etc/crowdsec/profiles.yaml
+    online_client: # Crowdsec API credentials (to push signals and receive bad IPs)
+      credentials_path: /etc/crowdsec/online_api_credentials.yaml
+#    tls:
+#      cert_file: /etc/crowdsec/ssl/cert.pem
+#      key_file: /etc/crowdsec/ssl/key.pem
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 127.0.0.1
+  listen_port: 6060

config/crowdsec.service

@@ -1,13 +1,13 @@
 [Unit]
-Description=Crowdwatch agent
+Description=Crowdsec agent
 After=syslog.target network.target remote-fs.target nss-lookup.target
 
 [Service]
-Type=forking
-PIDFile=${PID}/crowdsec.pid
-#ExecStartPre=${BIN} -c ${CFG}/default.yaml -t
-ExecStart=${BIN} -c ${CFG}/default.yaml
-ExecStartPost=/bin/sleep 0.1
+Type=notify
+PIDFile=/var/run/crowdsec.pid
+ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
+ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml
+#ExecStartPost=/bin/sleep 0.1
 ExecReload=/bin/kill -HUP $MAINPID
 
 [Install]

config/dev.yaml

@@ -1,18 +1,43 @@
-working_dir: "."
-data_dir: "./data"
-config_dir: "./config"
-pid_dir: "./"
-cscli_dir: "./config/crowdsec-cli"
-log_dir: "./logs"
-log_mode: "stdout"
-log_level: info
-prometheus: false
-simulation_path: ./config/simulation.yaml
-profiling: false
-apimode: false
-plugin:
-  backend: "./config/plugins/backend"
-  max_records: 10000
-  #30 days = 720 hours
-  max_records_age: 720h
-  
+common:
+  daemonize: true
+  pid_dir: /tmp/
+  log_media: stdout
+  log_level: info
+  working_dir: .
+config_paths:
+  config_dir: ./config
+  data_dir: ./data/
+  #simulation_path: /etc/crowdsec/config/simulation.yaml
+  #hub_dir: /etc/crowdsec/hub/
+  #index_path: ./config/hub/.index.json
+crowdsec_service:
+  #acquisition_path: ./config/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+db_config:
+  type: sqlite
+  db_path: ./data/crowdsec.db
+  user: root
+  password: crowdsec
+  db_name: crowdsec
+  host: "172.17.0.2"
+  port: 3306
+  flush:
+    #max_items: 10000
+    #max_age: 168h
+api:
+  client:
+    credentials_path: ./config/local_api_credentials.yaml
+  server:
+    #insecure_skip_verify: true
+    listen_uri: localhost:8081
+    profiles_path: ./config/profiles.yaml
+    tls:
+      #cert_file: ./cert.pem
+      #key_file: ./key.pem
+    online_client: # Crowdsec API
+      credentials_path: ./config/online_api_credentials.yaml
+prometheus:
+  enabled: true
+  level: full

config/local_api_credentials.yaml

@@ -0,0 +1 @@
+url: http://localhost:8080

config/plugins/backend/database.yaml

@@ -1,19 +0,0 @@
-name: database
-path: /usr/local/lib/crowdsec/plugins/backend/database.so
-config:
-  ## DB type supported (mysql, sqlite)
-  ## By default it using sqlite
-  type: sqlite
-
-  ## mysql options
-  # db_host: localhost
-  # db_username: crowdsec
-  # db_password: crowdsec
-  # db_name: crowdsec
-
-  ## sqlite options
-  db_path: /var/lib/crowdsec/data/crowdsec.db
-
-  ## Other options
-  flush: true
-  # debug: true

config/prod.yaml

@@ -1,18 +0,0 @@
-working_dir: /tmp/
-data_dir: ${DATA}
-config_dir: ${CFG}
-pid_dir: ${PID}
-log_dir: /var/log/
-cscli_dir: ${CFG}/cscli
-simulation_path: ${CFG}/simulation.yaml
-log_mode: file
-log_level: info
-profiling: false
-apimode: true
-daemon: true
-prometheus: true
-#for prometheus agent / golang debugging
-http_listen: 127.0.0.1:6060
-plugin:
-  backend: "/etc/crowdsec/plugins/backend"
-  max_records_age: 720h

config/profiles.yaml

@@ -1,26 +1,8 @@
-profile: default_remediation
-filter: "sig.Labels.remediation == 'true' && not sig.Whitelisted"
-api: true # If no api: specified, will use the default config in default.yaml
-remediation:
-  ban: true
-  slow: true
-  captcha: true
-  duration: 4h
-outputs:
-  - plugin: database
----
-profile: default_notification
-filter: "sig.Labels.remediation != 'true'"
-#remediation is empty, it means non taken
-api: false
-outputs:
-  - plugin: database  # If we do not want to push, we can remove this line and the next one
-    store: false
----
-profile: send_false_positif_to_API
-filter: "sig.Whitelisted == true && sig.Labels.remediation == 'true'"
-#remediation is empty, it means non taken
-api: true
-outputs:
-  - plugin: database  # If we do not want to push, we can remove this line and the next one
-    store: false
+name: default_ip_remediation
+#debug: true
+filters:
+ - Alert.Remediation == true && Alert.GetScope() == "Ip"
+decisions:
+ - type: ban
+   duration: 1h
+on_success: break

config/user.yaml

@@ -1,16 +1,40 @@
-working_dir: /tmp/
-data_dir: ${DATA}
-config_dir: ${CFG}
-pid_dir: ${PID}
-log_dir: /var/log/
-cscli_dir: ${CFG}/cscli
-log_mode: stdout
-log_level: info
-profiling: false
-apimode: false
-daemon: false
-prometheus: false
-#for prometheus agent / golang debugging
-http_listen: 127.0.0.1:6060
-plugin:
-  backend: "/etc/crowdsec/plugins/backend"
+common:
+  daemonize: false
+  pid_dir: /var/run/
+  log_media: stdout
+  log_level: info
+  log_dir: /var/log/
+  working_dir: .
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data
+  #simulation_path: /etc/crowdsec/config/simulation.yaml
+  #hub_dir: /etc/crowdsec/hub/
+  #index_path: ./config/hub/.index.json
+crowdsec_service:
+  #acquisition_path: ./config/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+db_config:
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  user: crowdsec
+  #log_level: info
+  password: crowdsec
+  db_name: crowdsec
+  host: "127.0.0.1"
+  port: 3306
+api:
+  client:
+    insecure_skip_verify: true # default true
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    #log_level: info
+    listen_uri: localhost:8080
+    profiles_path: /etc/crowdsec/profiles.yaml
+    online_client: # Crowdsec API
+      credentials_path: /etc/crowdsec/online_api_credentials.yaml
+prometheus:
+  enabled: true
+  level: full

docker/README.md

@@ -0,0 +1,129 @@
+# Crowdsec
+
+Crowdsec - An open-source, lightweight agent to detect and respond to bad behaviours. It also automatically benefits from our global community-wide IP reputation database.
+
+## Getting Started
+
+Before starting using docker image, we suggest you to read our documentation to understand all [crowdsec concepts](https://docs.crowdsec.net/).
+
+### Prerequisities
+
+
+In order to run this container you'll need docker installed.
+
+* [Windows](https://docs.docker.com/windows/started)
+* [OS X](https://docs.docker.com/mac/started/)
+* [Linux](https://docs.docker.com/linux/started/)
+
+### How to use ?
+
+#### Build
+
+```shell
+git clone https://github.com/crowdsecurity/crowdsec.git && cd crowdsec
+docker build -t crowdsec .
+```
+
+#### Run
+
+The container is built with specific docker [configuration](https://github.com/crowdsecurity/crowdsec/blob/master/docker/config.yaml) :
+
+You should apply following configuration before starting it :
+
+* Specify collections|scenarios|parsers/postoverflows to install via the environment variables (by default [`crowdsecurity/linux`](https://hub.crowdsec.net/author/crowdsecurity/collections/linux) is installed)
+* Mount volumes to specify your configuration
+* Mount volumes to specify your log files that should be ingested by crowdsec (set up in acquis.yaml)
+* Mount other volumes : if you want to share the database for example
+
+```shell
+docker run -d -v config.yaml:/etc/crowdsec/config.yaml \
+    -v acquis.yaml:/etc/crowdsec/acquis.yaml \
+    -e COLLECTIONS="crowdsecurity/sshd"
+    -v /var/log/auth.log:/var/log/auth.log \
+    -v /path/mycustom.log:/var/log/mycustom.log \
+    --name crowdsec <built-image-tag>
+```
+
+#### Example
+
+I have my own configuration :
+```shell
+user@cs ~/crowdsec/config $ ls
+acquis.yaml  config.yaml
+```
+
+Here is my acquis.yaml file:
+```shell
+filenames:
+ - /logs/auth.log
+ - /logs/syslog
+labels:
+  type: syslog
+---
+filename: /logs/apache2/*.log
+labels:
+  type: apache2
+```
+
+So, I want to run crowdsec with :
+
+* My configuration files
+* Ingested my path logs specified in acquis.yaml
+* Share the crowdsec sqlite database with my host (You need to create empty file first, otherwise docker will create a directory instead of simple file)
+* Expose local API through host (listen by default on `8080`)
+* Expose prometheus handler through host (listen by default on `6060`)
+
+```shell
+touch /path/myDatabase.db
+docker run -d -v config.yaml:/etc/crowdsec/config.yaml \
+    -v acquis.yaml:/etc/crowdsec/acquis.yaml \
+    -v /var/log/auth.log:/logs/auth.log \
+    -v /var/log/syslog.log:/logs/syslog.log \
+    -v /var/log/apache:/logs/apache \
+    -v /path/myDatabase.db:/var/lib/crowdsec/data/crowdsec.db \
+    -e COLLECTIONS="crowdsecurity/apache2 crowdsecurity/sshd" \
+    -p 8080:8080 -p 6060:6060 \
+    --name crowdsec <built-image-tag>
+```
+
+### Environment Variables
+
+* `COLLECTIONS`             - Collections to install from the [hub](https://hub.crowdsec.net/browse/#collections), separated by space : `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"`
+* `SCENARIOS`               - Scenarios to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e SCENARIOS="crowdsecurity/http-bad-user-agent crowdsecurity/http-xss-probing"`
+* `PARSERS`                 - Parsers to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e PARSERS="crowdsecurity/http-logs crowdsecurity/modsecurity"`
+* `POSTOVERFLOWS`           - Postoverflows to install from the [hub](https://hub.crowdsec.net/browse/#configurations), separated by space : `-e POSTOVERFLOWS="crowdsecurity/cdn-whitelist"`
+* `CONFIG_FILE`             - Configuration file (default: `/etc/crowdsec/config.yaml`) : `-e CONFIG_FILE="<config_path>"`
+* `FILE_PATH`               - Process a single file in time-machine : `-e FILE_PATH="<file_path>"`
+* `JOURNALCTL_FILTER`       - Process a single journalctl output in time-machine : `-e JOURNALCTL_FILTER="<journalctl_filter>"`
+* `TYPE`                    - [`Labels.type`](https://https://docs.crowdsec.net/Crowdsec/v1/references/acquisition/) for file in time-machine : `-e TYPE="<type>"`
+* `TEST_MODE`               - Only test configs (default: `false`) : `-e TEST_MODE="<true|false>"`
+* `DISABLE_AGENT`           - Only test configs (default: `false`) : `-e DISABLE_AGENT="<true|false>"`
+* `DISABLE_LOCAL_API`       - Disable local API (default: `false`) : `-e DISABLE_API="<true|false>"`
+* `REGISTER_TO_ONLINE_API`  - Register to Online API (default: `false`) : `-e REGISTER_TO_ONLINE_API="<true|false>"`
+* `LEVEL_TRACE`             - Trace-level (VERY verbose) on stdout (default: `false`) : `-e LEVEL_TRACE="<true|false>"`
+* `LEVEL_DEBUG`             - Debug-level on stdout (default: `false`) : `-e LEVEL_DEBUG="<true|false>"`
+* `LEVEL_INFO`              - Info-level on stdout (default: `false`) : `-e LEVEL_INFO="<true|false>"`
+
+### Volumes
+
+* `/var/lib/crowdsec/data/` - Directory where all crowdsec data (Databases) is located
+
+* `/etc/crowdsec/` - Directory where all crowdsec configurations are located
+
+#### Useful File Locations
+
+* `/usr/local/bin/crowdsec` - Crowdsec binary
+  
+* `/usr/local/bin/cscli` - Crowdsec CLI binary to interact with crowdsec
+
+## Find Us
+
+* [GitHub](https://github.com/crowdsecurity/crowdsec)
+
+## Contributing
+
+Please read [contributing](https://docs.crowdsec.net/Crowdsec/v1/contributing/) for details on our code of conduct, and the process for submitting pull requests to us.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](https://github.com/crowdsecurity/crowdsec/blob/master/LICENSE) file for details.

docker/config.yaml

@@ -0,0 +1,49 @@
+common:
+  daemonize: false
+  pid_dir: /var/run/
+  log_media: stdout
+  log_level: info
+  log_dir: /var/log/
+  working_dir: .
+config_paths:
+  config_dir: /etc/crowdsec/
+  data_dir: /var/lib/crowdsec/data/
+  simulation_path: /etc/crowdsec/simulation.yaml
+  hub_dir: /etc/crowdsec/hub/
+  index_path: /etc/crowdsec/hub/.index.json
+crowdsec_service:
+  acquisition_path: /etc/crowdsec/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+  hub_branch: wip_lapi
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: /var/lib/crowdsec/data/crowdsec.db
+  #user: 
+  #password:
+  #db_name:
+  #host:
+  #port:
+  flush:
+    max_items: 5000
+    max_age: 7d
+api:
+  client:
+    insecure_skip_verify: true
+    credentials_path: /etc/crowdsec/local_api_credentials.yaml
+  server:
+    log_level: info
+    listen_uri: 0.0.0.0:8080
+    profiles_path: /etc/crowdsec/profiles.yaml
+    online_client: # Crowdsec API credentials (to push signals and receive bad IPs)
+      #credentials_path: /etc/crowdsec/online_api_credentials.yaml
+#    tls:
+#      cert_file: /etc/crowdsec/ssl/cert.pem
+#      key_file: /etc/crowdsec/ssl/key.pem
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 0.0.0.0
+  listen_port: 6060

docker/docker_start.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+# Check if the container has already been started
+cscli machines list | grep 127.0.0.1
+if [ $? == 1 ]; then
+    cscli machines add --force --auto -f /etc/crowdsec/local_api_credentials.yaml
+fi
+
+if [ "$REGISTER_TO_ONLINE_API" == "true" ] || [ "$REGISTER_TO_ONLINE_API" == "TRUE" ] && [ "$CONFIG_FILE" == "" ] ; then
+    cat /etc/crowdsec/config.yaml | grep online_api_credentials.yaml
+    if [ $? == 1 ]; then
+        sed -ri 's/^(\s*)(#credentials_path\s*:\s*$)/\1credentials_path: \/etc\/crowdsec\/online_api_credentials.yaml/' /etc/crowdsec/config.yaml
+        cscli capi register > /etc/crowdsec/online_api_credentials.yaml
+    fi
+fi
+
+## Install collections, parsers & scenarios
+cscli hub update
+cscli collections upgrade crowdsecurity/linux
+if [ "$COLLECTIONS" != "" ]; then
+    cscli collections install $COLLECTIONS
+fi
+if [ "$PARSERS" != "" ]; then
+    cscli parsers install $PARSERS
+fi
+if [ "$SCENARIOS" != "" ]; then
+    cscli scenarios install $SCENARIOS
+fi
+if [ "$POSTOVERFLOWS" != "" ]; then
+    cscli postoverflows install $POSTOVERFLOWS
+fi
+
+ARGS=""
+if [ "$CONFIG_FILE" != "" ]; then
+    ARGS="-c $CONFIG_FILE"
+fi
+if [ "$FILE_PATH" != "" ]; then
+    ARGS="$ARGS -file $FILE"
+fi
+if [ "$JOURNALCTL_FILTER" != "" ]; then
+    ARGS="$ARGS -jfilter $JOURNALCTL_FILTER"
+fi
+if [ "$TYPE" != "" ]; then
+    ARGS="$ARGS -type $TYPE"
+fi
+if [ "$TEST_MODE" == "true" ] || [ "$TEST_MODE" == "TRUE" ]; then
+    ARGS="$ARGS -t"
+fi
+if [ "$DISABLE_AGENT" == "true" ] || [ "$DISABLE_AGENT" == "TRUE" ]; then
+    ARGS="$ARGS -no-cs"
+fi
+if [ "$DISABLE_API" == "true" ] || [ "$DISABLE_API" == "TRUE" ]; then
+    ARGS="$ARGS -no-api"
+fi
+if [ "$LEVEL_TRACE" == "true" ] || [ "$LEVEL_TRACE" == "TRUE" ]; then
+    ARGS="$ARGS -trace"
+fi
+if [ "$LEVEL_DEBUG" == "true" ] || [ "$LEVEL_DEBUG" == "TRUE"  ]; then
+    ARGS="$ARGS -debug"
+fi
+if [ "$LEVEL_INFO" == "true" ] || [ "$LEVEL_INFO" == "TRUE" ]; then
+    ARGS="$ARGS -info"
+fi
+
+exec crowdsec $ARGS

docs/about.md

@@ -1,3 +0,0 @@
-# Crowdsec
-
-{{ macros_info() }}

docs/bouncers/index.md

@@ -1,12 +0,0 @@
-# bouncers
-
-
-{{bouncers.Name}} are standalone software pieces in charge of acting upon blocked IPs.
-
-They can either be within the applicative stack, or work out of band :
-
-[nginx bouncer](https://github.com/crowdsecurity/cs-nginx-bouncer) will check every unknown IP against the database before letting go through or serving a *403* to the user, while a [netfilter bouncer](https://github.com/crowdsecurity/cs-netfilter-bouncer) will simply "add" malevolent IPs to nftables/ipset set of blacklisted IPs.
-
-
-You can explore [available {{bouncers.name}} on the hub]({{hub.plugins_url}}), and find below a few of the "main" {{bouncers.name}}.
-

docs/faq.md

@@ -0,0 +1,135 @@
+# FREQUENTLY ASKED QUESTIONS
+
+## What is {{v1X.crowdsec.name}} ?
+
+{{v1X.crowdsec.Name}} is a security open-source software. See the [overview](/#what-is-crowdsec)
+
+
+## What language is it written in ?
+
+{{v1X.crowdsec.Name}} is written in [Golang](https://golang.org/) 
+
+## What licence is {{v1X.crowdsec.name}} released under ?
+
+{{v1X.crowdsec.Name}} is under [MIT license]({{v1X.crowdsec.url}}/blob/master/LICENSE)
+
+## Which information is sent to the APIs ?
+
+Our aim is to build a strong community that can share malevolent attackers IPs, for that we need to collect the bans triggered locally by each user.
+
+The signal sent by your {{v1X.crowdsec.name}} to the central API only contains only meta-data about the attack :
+
+ - Attacker IP
+ - Scenario name
+ - Time of start/end of attack
+
+Your logs are not sent to our central API, only meta-data about blocked attacks will be.
+
+## What is the performance impact ?
+
+As {{v1X.crowdsec.name}} only works on logs, it shouldn't impact your production.
+When it comes to {{v1X.bouncers.name}}, it should perform **one** request to the database when a **new** IP is discovered thus have minimal performance impact.
+
+## How fast is it ?
+
+{{v1X.crowdsec.name}} can easily handle several thousands of events per second on a rich pipeline (multiple parsers, geoip enrichment, scenarios and so on). Logs are a good fit for sharding by default, so it is definitely the way to go if you need to handle higher throughput.
+
+If you need help for large scale deployment, please get in touch with us on the {{v1X.doc.discourse}}, we love challenges ;)
+
+## What backend database does {{v1X.crowdsec.Name}} supports and how to switch ?
+
+{{v1X.crowdsec.name}} versions (under v0.3.X) supports SQLite (default) and MySQL databases.
+See [backend configuration](/Crowdsec/v0/references/output/#switching-backend-database) for relevant configuration. MySQL here is more suitable for distributed architectures where bouncers across the applicative stack need to access a centralized ban database.
+
+{{v1X.crowdsec.name}} versions (after v1) supports SQLite (default), MySQL and PostgreSQL databases.
+See [databases configuration](/Crowdsec/v1/user_guide/database/) for relevant configuration. Thanks to the {{v1X.lapi.Htmlname}}, distributed architectures are resolved even with sqlite database.
+
+SQLite by default as it's suitable for standalone/single-machine setups.
+
+## How to control granularity of actions ? (whitelists, simulation etc.)
+
+{{v1X.crowdsec.name}} support both [whitelists]((/Crowdsec/v1/write_configurations/whitelist/) and [simulation](/Crowdsec/v1/references/simulation/) :
+
+ - Whitelists allows you to "discard" events or overflows
+ - Simulation allows you to simply cancel the decision that is going to be taken, but keep track of it
+
+## How to add whitelists ?
+
+You can follow this [guide](/Crowdsec/v1/write_configurations/whitelist/)
+
+## How to set up proxy ?
+
+Setting up a proxy works out of the box, the [net/http golang library](https://golang.org/src/net/http/transport.go) can handle those environment variables:
+
+* `HTTP_PROXY`
+* `HTTPS_PROXY`
+* `NO_PROXY`
+
+Since {{v1X.cli.name}} uses `sudo`, you just this line in `visudo` after setting up the previous environment variables:
+
+```
+Defaults env_keep += "HTTP_PROXY HTTPS_PROXY NO_PROXY"
+```
+
+## How to report a bug ?
+
+To report a bug, please open an issue on the [repository]({{v1X.crowdsec.bugreport}})
+
+## What about false positives ?
+
+Several initiatives have been taken to tackle the false positives approach as early as possible :
+
+ - The scenarios published on the hub are tailored to favor low false positive rates
+ - You can find [generic whitelists](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors) that should allow to cover most common cases (SEO whitelists, CDN whitelists etc.)
+ - The [simulation configuration](/Crowdsec/v1/references/simulation/) allows you to keep a tight control over scenario and their false positives
+
+
+## I need some help
+
+Feel free to ask for some help to the {{v1X.doc.community}}.
+
+## How to use crowdsec on raspberry pi OS (formerly known as rasbian) 
+
+Please keep in mind that raspberry pi OS is designed to work on all
+raspberry pi versions. Even if the port target is known as armhf, it's
+not exactly the same target as the debian named armhf port.
+
+The best way to have a crowdsec version for such an architecture is to
+do:
+1. install golang (all versions from 1.13 will do)
+2. `export GOARCH=arm`
+3. `export CGO=1`
+4. Update the GOARCH variable in the Makefile to `arm`
+5. install the arm gcc cross compilator (On debian the package is gcc-arm-linux-gnueabihf)
+6. Compile crowdsec using the usual `make` command
+
+
+<!-- 
+
+## How to contribute ?
+
+### On {{v1X.crowdsec.Name}}
+
+### On Configurations (Parsers, scenarios)
+
+### On bouncers
+
+
+
+## What are common use-cases ?
+
+**TBD**
+
+## What about false positives ?
+
+**TBD**
+
+## How to test if it works ?
+
+**TBD**
+
+## Who are you ?
+
+**TBD**
+
+-->

docs/getting_started/FAQ.md

@@ -1,137 +0,0 @@
-# FREQUENTLY ASKED QUESTIONS
-
-## What is {{crowdsec.name}} ?
-
-{{crowdsec.Name}} is a security open-source software. See the [overview](/#what-is-crowdsec)
-
-
-## What language is it written in ?
-
-{{crowdsec.Name}} is written in [Golang](https://golang.org/) 
-
-## What licence is {{crowdsec.name}} released under ?
-
-{{crowdsec.Name}} is under [MIT license]({{crowdsec.url}}/blob/master/LICENSE)
-
-## Which information is sent to the APIs ?
-
-Our aim is to build a strong community that can share malevolent attackers IPs, for that we need to collect the bans triggered locally by each user.
-
-The alert sent by your {{crowdsec.name}} to the central API only contains only meta-data about the attack :
-
- - Attacker IP
- - Scenario name
- - Time of start/end of attack
-
-Your logs are not sent to our central API, only meta-data about blocked attacks will be.
-
-When your crowdsec is authenticating to the central API, it as well sends the list of the scenarios you have enabled from the hub. This is used by us to provide you the most accurate consensus list, so that we can provide you IPs that have triggered scenario(s) that you are interested into as well.
-
-
-## What is the performance impact ?
-
-As {{crowdsec.name}} only works on logs, it shouldn't impact your production.
-When it comes to {{bouncers.name}}, it should perform **one** request to the database when a **new** IP is discovered thus have minimal performance impact.
-
-## How fast is it ?
-
-{{crowdsec.name}} can easily handle several thousands of events per second on a rich pipeline (multiple parsers, geoip enrichment, scenarios and so on). Logs are a good fit for sharding by default, so it is definitely the way to go if you need to handle higher throughput.
-
-If you need help for large scale deployment, please get in touch with us on the {{doc.discourse}}, we love challenges ;)
-
-## What backend database does {{crowdsec.Name}} supports and how to switch ?
-
-Currently (0.3.0), {{crowdsec.name}} supports SQLite (default) and MySQL databases.
-See [backend configuration](/references/output/#switching-backend-database) for relevant configuration.
-
-SQLite is the default backend as it's suitable for standalone/single-machine setups.
-On the other hand, MySQL is more suitable for distributed architectures where bouncers across the applicative stack need to access a centralized ban database.
-
-## How to control granularity of actions ? (whitelists, learning etc.)
-
-{{crowdsec.name}} support both [whitelists]((/write_configurations/whitelist/) and [learning](/guide/crowdsec/simulation/) :
-
- - Whitelists allows you to "discard" events or overflows
- - Learning allows you to simply cancel the decision that is going to be taken, but keep track of it
-
-## How to add whitelists ?
-
-You can follow this [guide](/write_configurations/whitelist/)
-
-## How to set up proxy ?
-
-Setting up a proxy works out of the box, the [net/http golang library](https://golang.org/src/net/http/transport.go) can handle those environment variables:
-
-* `HTTP_PROXY`
-* `HTTPS_PROXY`
-* `NO_PROXY`
-
-Since {{cli.name}} uses `sudo`, you just this line in `visudo` after setting up the previous environment variables:
-
-```
-Defaults env_keep += "HTTP_PROXY HTTPS_PROXY NO_PROXY"
-```
-
-## How to report a bug ?
-
-To report a bug, please open an issue on the [repository]({{crowdsec.bugreport}})
-
-## What about false positives ?
-
-Several initiatives have been taken to tackle the false positives approach as early as possible :
-
- - The scenarios published on the hub are tailored to favor low false positive rates
- - You can find [generic whitelists](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors) that should allow to cover most common cases (SEO whitelists, CDN whitelists etc.)
- - The [simulation configuration](/guide/crowdsec/simulation/) allows you to keep a tight control over scenario and their false positives
-
-
-## I need some help
-
-Feel free to ask for some help to the {{doc.community}}.
-
-## I don't see anything in the dashboard !
-
-Whenever in doubt with what is being processed or not, check [cscli metrics](/observability/command_line/). It should allow you to check :
- - if the logs are properly read
- - if the logs are properly parsed
- - if the scenarios are being triggered
- 
-If logs are being read, parsed and overflows are being triggered, but still nothing appears in the dashboard, ask for some help on discourse or gitter !
-
-
-## I have installed crowdsec and it detect attacks, but nothing is blocked !
-
-Keep in mind that {{crowdsec.Htmlname}} is only in charge of the detection. The decision/remediation is applied by {{bouncers.Htmlname}}.
-If you don't install any bouncer, you will detect attack, but not block them. Explore the [bouncers in the hub]({{bouncers.url}}) to find the relevant ones !
-
-
-
-<!-- 
-
-## How to contribute ?
-
-### On {{crowdsec.Name}}
-
-### On Configurations (Parsers, scenarios)
-
-### On bouncers
-
-
-
-## What are common use-cases ?
-
-**TBD**
-
-## What about false positives ?
-
-**TBD**
-
-## How to test if it works ?
-
-**TBD**
-
-## Who are you ?
-
-**TBD**
-
--->

docs/getting_started/installation.md

@@ -1,139 +0,0 @@
-# Installation
-
-Fetch {{crowdsec.name}}'s latest version [here]({{crowdsec.download_url}}).
-
-```bash
-tar xvzf crowdsec-release.tgz
-```
-```bash
-cd crowdsec-v0.X.X
-```
-
-A {{wizard.name}} is provided to help you deploy {{crowdsec.name}} and {{cli.name}}.
-
-## Using the interactive wizard
-
-```
-sudo {{wizard.bin}} -i
-```
-
-![crowdsec](../assets/images/crowdsec_install.gif)
-
-The {{wizard.name}} is going to guide you through the following steps :
-
- - detect services that are present on your machine
- - detect selected services logs
- - suggest collections (parsers and scenarios) to deploy
- - deploy & configure {{crowdsec.name}} in order to watch selected logs for selected scenarios
- 
-The process should take less than a minute, [please report if there are any issues]({{wizard.bugreport}}).
-
-You are then ready to [take a tour](/getting_started/crowdsec-tour/) of your freshly deployed {{crowdsec.name}} !
-
-## Binary installation
-
-> you of little faith
-
-```
-sudo {{wizard.bin}} --bininstall
-```
-
-This will deploy a valid/empty {{crowdsec.name}} configuration files and binaries.
-Beware, in this state, {{crowdsec.name}} won't monitor/detect anything unless configured.
-
-```
-cscli install collection crowdsecurity/linux
-```
-
-
-Installing at least the `crowdsecurity/linux` collection will provide you :
-
- - syslog parser
- - geoip enrichment
- - date parsers
-
-
-You will need as well to configure your {{ref.acquis}} file to feed {{crowdsec.name}} some logs.
-
-
-
-
-
-## From source
-
-!!! warning "Requirements"
-    
-    * [Go](https://golang.org/doc/install) v1.13+
-    * `git clone {{crowdsec.url}}`
-    * [jq](https://stedolan.github.io/jq/download/)
-
-
-Go in {{crowdsec.name}} folder and build the binaries :
-
-```bash
-cd crowdsec
-```
-```bash
-make build
-```
-
-
-{{crowdsec.name}} bin will be located in `./cmd/crowdsec/crowdsec` and {{cli.name}} bin in `cmd/crowdsec-cli/{{cli.bin}}` 
-
-Now, you can install either with [interactive wizard](#using-the-interactive-wizard) or the [unattended mode](#using-unattended-mode).
-
-
-# Upgrading
-
-The wizard itself comes with a `--upgrade` option, that will upgrade existing crowdsec version.
-
-If you have installed crowdsec `v0.1.0` and you downloaded `v0.1.1`, you can run `sudo ./wizard.sh --upgrade` from the extracted `v0.1.1` version. (_note_: the wizard doesn't *yet* download the latest version, you have to download it)
-
-
-The wizard takes care of backing up configurations on your behalf, and puts them into an archive :
-
- - backup your parsers,scenarios,collections, either from hub or your local ones
- - simulation configuration
- - API credentials
- - acquisition.yaml file
- - plugin(s) configuration
-
-It will then install the new/current crowdsec version, and restore everything that has been backed up!
-
-
-```bash
-$ sudo ./wizard.sh --upgrade
-[10/05/2020:11:27:34 AM][INF] crowdsec_wizard: Backing up existing configuration
-WARN[0000] Starting configuration backup                
-INFO[0000] saving, version:0.1, up-to-date:true          file=crowdsecurity/syslog-logs type=parsers
-...
-INFO[0000] Wrote 7 entries for parsers to /tmp/tmp.z54P27aaW0/parsers//upstream-parsers.json  file=crowdsecurity/geoip-enrich type=parsers
-INFO[0000] Wrote 0 entries for postoverflows to /tmp/tmp.z54P27aaW0/postoverflows//upstream-postoverflows.json  file=crowdsecurity/seo-bots-whitelist type=postoverflows
-INFO[0000] Wrote 9 entries for scenarios to /tmp/tmp.z54P27aaW0/scenarios//upstream-scenarios.json  file=crowdsecurity/smb-bf type=scenarios
-INFO[0000] Wrote 4 entries for collections to /tmp/tmp.z54P27aaW0/collections//upstream-collections.json  file=crowdsecurity/vsftpd type=collections
-INFO[0000] Saved acquis to /tmp/tmp.z54P27aaW0/acquis.yaml 
-INFO[0000] Saved default yaml to /tmp/tmp.z54P27aaW0/default.yaml 
-INFO[0000] Saved configuration to /tmp/tmp.z54P27aaW0   
-INFO[0000] Stop docker metabase /crowdsec-metabase      
-[10/05/2020:11:27:36 AM][INF] crowdsec_wizard: Removing crowdsec binaries
-[10/05/2020:11:27:36 AM][INF] crowdsec_wizard: crowdsec successfully uninstalled
-[10/05/2020:11:27:36 AM][INF] crowdsec_wizard: Installing crowdsec
-...
-[10/05/2020:11:27:36 AM][INF] crowdsec_wizard: Restoring configuration
-...
-INFO[0004] Restore acquis to /etc/crowdsec/config/acquis.yaml 
-INFO[0004] Restoring  '/tmp/tmp.z54P27aaW0/plugins/backend/database.yaml' to '/etc/crowdsec/plugins/backend/database.yaml' 
-[10/05/2020:11:27:41 AM][INF] crowdsec_wizard: Restoring saved database
-[10/05/2020:11:27:41 AM][INF] crowdsec_wizard: Finished, restarting
-
-```
-
-As usual, if you experience any issues, let us know :)
-
-# Uninstalling
-
-You can uninstall crowdsec using the wizard : `sudo ./wizard.sh --uninstall`
-
-
-
-

docs/getting_started/known_issues.md

@@ -1,15 +0,0 @@
-# Known bugs and limitations
-
-## Some users experience crash on 32bits architecture
-
-For now, on 32bit architecture there's a alignment bug in the way
-https://github.com/jamiealquiza/tachymeter library uses the [sync package](https://golang.org/pkg/sync/atomic/#pkg-note-BUG) that prevents crowdsec from running properly with prometheus gathering metrics.
-
-All versions  v0.3.X up to v0.3.5 are affected.
-
-The workaround is to disable prometheus until the bug is fixed.  For
-doing this you'll have to set `prometheus` to `false` in the file
-`/etc/crowdsec/config/default.yaml`.
-
-We are working on solving this issue by getting rid of the culprit
-library.

docs/guide/cscli.md

@@ -1,39 +0,0 @@
-`{{cli.bin}}` is the utility that will help you to manage {{crowdsec.name}}. This tools has the following functionalities:
-
- - [manage bans]({{ cli.ban_doc }})
- - [backup and restore configuration]({{ cli.backup_doc }})
- - [display metrics]({{ cli.metrics_doc }})
- - [install configurations]({{ cli.install_doc }})
- - [remove configurations]({{ cli.remove_doc }})
- - [update configurations]({{ cli.update_doc }})
- - [upgrade configurations]({{ cli.upgrade_doc }})
- - [list configurations]({{ cli.list_doc }})
- - [interact with CrowdSec API]({{ cli.api_doc }})
- - [manage simulation]({{cli.simulation_doc}})
-
- Take a look at the [dedicated documentation]({{cli.main_doc}})
-
-## Overview
-
-{{cli.name}} configuration location is `/etc/crowdsec/cscli/`. 
-
-In this folder, we store the {{cli.name}} configuration and the hub cache files.
-
-## Config
-
-The {{cli.name}} configuration is light for now, stored in `/etc/crowdsec/cscli/config`.
-
-```yaml
-installdir: /etc/crowdsec/config   # {{crowdsec.name}} configuration location
-backend: /etc/crowdsec/plugins/backend # path to the backend plugin used
-```
-
-For {{cli.name}} to be able to pull the {{api.topX.htmlname}}, you need a valid API configuration in [api.yaml](/guide/crowdsec/overview/#apiyaml).
-
-
-## Hub cache
-
-- `.index.json`: The file containing the metadata of all the existing {{collections.htmlname}}, {{parsers.htmlname}} and {{scenarios.htmlname}} stored in the {{hub.htmlname}}.
-- `hub/*`: Folder containing all the {{collections.htmlname}}, {{parsers.htmlname}} and {{scenarios.htmlname}} stored in the {{hub.htmlname}}.
-
-This is used to manage configurations from the {{cli.name}}

docs/guide/overview.md

@@ -1,10 +0,0 @@
-
-When talking about {{crowdsec.name}} or {{cli.name}} configurations, most of things are going to gravitate around {{parsers.htmlname}},  {{scenarios.htmlname}} and {{collections.htmlname}}.
-
-In most common setup, all these configurations should be found on the {{hub.htmlname}} and installed with {{cli.name}}.
-
-It is important to keep those configurations up-to-date via the `{{cli.name}} upgrade` command.
-
-See the [{{cli.name}} list](/cheat_sheets/cscli-collections-tour/) command to view the state of your deployed configurations.
-
-

docs/index.md

@@ -1,65 +1,41 @@
-<center>[[Hub]]({{hub.url}}) [[Releases]]({{crowdsec.download_url}})</center>
+<center>[[Hub]]({{v1X.hub.url}}) [[Releases]]({{v1X.crowdsec.download_url}})</center>
 
-# What is {{crowdsec.Name}} ?
 
-[{{crowdsec.Name}}]({{crowdsec.url}}) is an open-source and lightweight software that allows you to detect peers with malevolent behaviors and block them (using {{bouncers.Htmlname}}) from accessing your systems at various levels (infrastructural, system, applicative).
+!!! warning
+        For crowdsec versions `<= 1.0` please refer to [v0.3.X](/Crowdsec/v0/)
 
-To achieve this, {{crowdsec.Name}} reads logs from different sources (files, streams ...) to parse, normalize and enrich them before matching them to threats patterns called scenarios. 
+        For crowdsec versions `>= 1.0` please refer to [v1.X](/Crowdsec/v1/)
 
-{{crowdsec.Name}} is a modular and plug-able framework, it ships a large variety of [well known popular scenarios](https://hub.crowdsec.net/browse/#configurations); users can choose what scenarios they want to be protected from as well as easily adding new custom ones to better fit their environment.
+# What is {{v1X.crowdsec.Name}} ?
 
-Detected malevolent peers can then be prevented from accessing your resources by deploying [bouncers]({{hub.plugins_url}}) at various levels (applicative, system, infrastructural) of your stack.
+[{{v1X.crowdsec.Name}}]({{v1X.crowdsec.url}}) is an open-source and lightweight software that allows you to detect peers with malevolent behaviors and block them from accessing your systems at various level (infrastructural, system, applicative).
 
-One of the advantages of Crowdsec when compared to other solutions is its crowd-sourced aspect : Meta information about detected attacks (source IP, time and triggered scenario) are sent to a central API and then shared amongst all users.
-
-Thanks to this, besides detecting and stopping attacks in real time based on your logs, it allows you to preemptively block known bad actors from accessing your information system.
-
-
-## Components
-
-{{crowdsec.name}} ecosystem is based on the following components :
-
- - [{{crowdsec.Name}}]({{crowdsec.url}}) is the lightweight service that processes logs and keeps track of attacks.
- - [{{cli.name}}]({{cli.main_doc}}) is the command line interface for humans, it allows you to view, add, or remove bans as well as to install, find, or update scenarios and parsers
- - [{{bouncers.name}}]({{hub.plugins_url}}) are the components that block malevolent traffic, and can be deployed anywhere in your stack
+To achieve this, {{v1X.crowdsec.Name}} reads logs from different sources (files, streams ...) to parse, normalize and enrich them before matching them to threats patterns called scenarios. 
 
-## Architecture
+{{v1X.crowdsec.Name}} is a modular and plug-able framework, it ships a large variety of [well known popular scenarios](https://hub.crowdsec.net/browse/#configurations); users can choose what scenarios they want to be protected from as well as easily adding new custom ones to better fit their environment.
 
-![Architecture](assets/images/crowdsec_architecture.png)
+Detected malevolent peers can then be prevented from accessing your resources by deploying [bouncers]({{v1X.hub.bouncers_url}}) at various levels (applicative, system, infrastructural) of your stack.
 
+One of the advantages of Crowdsec when compared to other solutions is its crowd-sourced aspect : Meta information about detected attacks (source IP, time and triggered scenario) are sent to a central API and then shared amongst all users.
 
-## Core concepts
-
-{{crowdsec.name}} relies on {{parsers.htmlname}} to normalize and enrich logs, and {{scenarios.htmlname}} to detect attacks, often bundled together in {{collections.htmlname}} to form a coherent configuration set. For example the collection [`crowdsecurity/nginx`](https://hub.crowdsec.net/author/crowdsecurity/collections/nginx) contains all the necessary parsers and scenarios to deal with nginx logs and the common attacks that can be seen on http servers.
-
-All of those are represented as YAML files, that can be found, shared and kept up-to-date thanks to the {{hub.htmlname}}, or [easily hand-crafted](/write_configurations/scenarios/) to address specific needs.
+Thanks to this, besides detecting and stopping attacks in real time based on your logs, it allows you to preemptively block known bad actors from accessing your information system.
 
 
 ## Main features
 
-{{crowdsec.Name}}, besides the core "detect and react" mechanism,  is committed to a few other key points :
-
- - **Easy Installation** : The provided wizard allows a [trivial deployment](/getting_started/installation/#using-the-interactive-wizard) on most standard setups
- - **Easy daily operations** : Using [cscli](/cscli/cscli_upgrade/) and the {{hub.htmlname}}, keeping your detection mechanisms up-to-date is trivial
- - **Observability** : Providing strongs insights on what is going on and what {{crowdsec.name}} is doing :
-    - Humans have [access to a trivially deployable web interface](/observability/dashboard/)
-    - OPs have [access to detailed prometheus metrics](/observability/prometheus/)
-    - Admins have [a friendly command-line interface tool](/observability/command_line/) 
-
-## Moving forward
+{{v0X.crowdsec.Name}}, besides the core "detect and react" mechanism,  is committed to a few other key points :
 
-To learn more about {{crowdsec.name}} and give it a try, please see :
+ - **Easy Installation** : The provided wizard allows a [trivial deployment](/Crowdsec/v1/getting_started/installation/#using-the-interactive-wizard) on most standard setups
+ - **Easy daily operations** : Using [cscli](/Crowdsec/v1/cscli/cscli_upgrade/) and the {{v0X.hub.htmlname}}, keeping your detection mechanisms up-to-date is trivial
+ - **Reproducibility** : Crowdsec can run not only against live logs, but as well against cold logs. It makes it a lot easier to detect potential false-positives, perform forensic ou generate reporting
+ - **Observability** : Providing strongs insights on what is going on and what {{v0X.crowdsec.name}} is doing :
+    - Humans have [access to a trivially deployable web interface](/Crowdsec/v1/observability/dashboard/)
+    - OPs have [access to detailed prometheus metrics](/Crowdsec/v1/observability/prometheus/)
+    - Admins have [a friendly command-line interface tool](/Crowdsec/v1/observability/command_line/)
 
- - [How to install {{crowdsec.name}}](/getting_started/installation/)
- - [Take a quick tour of {{crowdsec.name}} and {{cli.name}} features](/getting_started/crowdsec-tour/)
- - [Observability of {{crowdsec.name}}](/observability/overview/)
- - [Understand {{crowdsec.name}} configuration](/getting_started/concepts/)
- - [Deploy {{bouncers.name}} to stop malevolent peers](/bouncers/)
- - [FAQ](getting_started/FAQ/)
- - [Known bugs and limitations](/getting_started/known_issues)
+## About this documentation
 
-Don't hesitate to reach out if you're facing issues :
+This document is split according to major {{v1X.crowdsec.Name}} versions :
 
- - [report a bug](https://github.com/crowdsecurity/crowdsec/issues/new?assignees=&labels=bug&template=bug_report.md&title=Bug%2F)
- - [suggest an improvement](https://github.com/crowdsecurity/crowdsec/issues/new?assignees=&labels=enhancement&template=feature_request.md&title=Improvment%2F)
- - [ask for help on the forums](https://discourse.crowdsec.net)
+ - [Crowdsec v0](/Crowdsec/v0/) Refers to versions `0.3.X`, before the local API was introduced. (_note: this is going to be deprecated and your are strongly incited to migrate to versions 1.X_)
+ - [Crowdsec v1](/Crowdsec/v1/) Refers to versions `1.X`, it is the current version

docs/observability/overview.md

@@ -1,13 +0,0 @@
-# Observability Overview
-
-Observability in security software is crucial, especially when this software might take important decision such as blocking IP addresses.
-
-We attempt to provide good observability of {{crowdsec.name}}'s behavior :
-
- - {{crowdsec.name}} itself exposes a [prometheus instrumentation](/observability/prometheus/)
- - {{cli.Name}} allows you to view part of prometheus metrics in [cli (`{{cli.bin}} metrics`)](/observability/command_line/)
- - {{crowdsec.name}} logging is contextualized for easy processing
- - for **humans**, {{cli.name}} allows you to trivially start a service [exposing dashboards](/observability/dashboard/) (using [metabase](https://www.metabase.com/))
-
-Furthermore, most of {{crowdsec.name}} configuration should allow you to enable partial debug (ie. per-scenario, per-parser etc.)
-

docs/requirements.txt

@@ -1,24 +1,27 @@
--i https://pypi.org/simple/
 click==7.1.1
 future==0.18.2
-jinja2==2.11.1
+Jinja2==2.11.1
 joblib==0.14.1
 livereload==2.6.1
-lunr[languages]==0.5.6
-markdown==3.2.1
-markupsafe==1.1.1
-mkdocs-material==4.6.3
+lunr==0.5.6
+Markdown==3.2.1
+MarkupSafe==1.1.1
 mkdocs==1.1
-mkdocs-macros-plugin==0.4.6
+mkdocs-macros-plugin==0.4.18
+mkdocs-material==6.1.0
+mkdocs-material-extensions==1.0.1
+mkdocs-monorepo-plugin==0.4.11
+mkdocs-redirects==1.0.1
 nltk==3.5b1
-prompt-toolkit==3.0.5
-pygments==2.6.1
-pymdown-extensions==7.0rc1
+prompt-toolkit==2.0.10
+Pygments==2.6.1
+pymdown-extensions==7.0
 python-markdown-math==0.6
-pyyaml==5.3.1
+PyYAML==5.3.1
 regex==2020.2.20
+repackage==0.7.3
 six==1.14.0
+termcolor==1.1.0
 tornado==6.0.4
 tqdm==4.43.0
 wcwidth==0.1.9
-

docs/v0.3.X/docs/bouncers/index.md

@@ -0,0 +1,12 @@
+# bouncers
+
+
+{{v0X.bouncers.Name}} are standalone software pieces in charge of acting upon blocked IPs.
+
+They can either within the applicative stack, or work out of band :
+
+[nginx blocker](https://github.com/crowdsecurity/cs-nginx-blocker) will check every unknown IP against the database before letting go through or serving a *403* to the user, while a [netfilter blocker](https://github.com/crowdsecurity/cs-netfilter-blocker) will simply "add" malevolent IPs to nftables/ipset set of blacklisted IPs.
+
+
+You can explore [available {{v0X.bouncers.name}} on the hub]({{v0X.hub.plugins_url}}), and find below a few of the "main" {{v0X.bouncers.name}} :
+

docs/cheat_sheets/ban-mgmt.md → docs/v0.3.X/docs/cheat_sheets/ban-mgmt.md

@@ -1,11 +1,11 @@
 !!! info 
 
-    Please see your local `{{cli.bin}} help ban` for up-to-date documentation.
+    Please see your local `{{v0X.cli.bin}} help ban` for up-to-date documentation.
 
 ## List bans
 
 ```bash
-{{cli.bin}} ban list
+{{v0X.cli.bin}} ban list
 ```
 
 <details>
@@ -28,8 +28,8 @@ And 64 records from API, 32 distinct AS, 19 distinct countries
 </details>
 
  - `SOURCE` is the source of the decision :
-    - "local" : the decision has been taken by {{crowdsec.name}}
-    - "cli" : the decision has been made with {{cli.name}} (ie. `{{cli.name}} ban ip 1.2.3.4 24h "because"`)
+    - "local" : the decision has been taken by {{v0X.crowdsec.name}}
+    - "cli" : the decision has been made with {{v0X.cli.name}} (ie. `{{v0X.cli.name}} ban ip 1.2.3.4 24h "because"`)
     - "api" : the decision has been pushed to you by the API (because there is a consensus about this ip)
  - `IP` is the IP or the IP range impacted by the decision
  - `REASON` is the scenario that was triggered (or human-supplied reason)
@@ -38,7 +38,7 @@ And 64 records from API, 32 distinct AS, 19 distinct countries
  - `EXPIRATION` is the time left on remediation
 
 
-Check [command usage](/cscli/cscli_ban_list/) for additional filtering and output control flags.
+Check [command usage](/Crowdsec/v0/cscli/cscli_ban_list/) for additional filtering and output control flags.
 
 
 ## Delete a ban
@@ -46,13 +46,13 @@ Check [command usage](/cscli/cscli_ban_list/) for additional filtering and outpu
 > delete the ban on IP `1.2.3.4`
 
 ```bash
-{{cli.bin}} ban del ip 1.2.3.4
+{{v0X.cli.bin}} ban del ip 1.2.3.4
 ```
 
 > delete the ban on range 1.2.3.0/24
 
 ```bash
-{{cli.bin}} ban del range 1.2.3.0/24
+{{v0X.cli.bin}} ban del range 1.2.3.0/24
 ```
 
 
@@ -61,13 +61,13 @@ Check [command usage](/cscli/cscli_ban_list/) for additional filtering and outpu
 > Add a ban on IP  `1.2.3.4` for 24 hours, with reason 'web bruteforce'
 
 ```bash
-{{cli.bin}} ban add ip 1.2.3.4 24h "web bruteforce"
+{{v0X.cli.bin}} ban add ip 1.2.3.4 24h "web bruteforce"
 ```
 
 > Add a ban on range  `1.2.3.0/24` for 24 hours, with reason 'web bruteforce'
 
 ```bash
-{{cli.bin}} ban add range 1.2.3.0/24 "web bruteforce"
+{{v0X.cli.bin}} ban add range 1.2.3.0/24 "web bruteforce"
 ```
 
 
@@ -77,7 +77,7 @@ Check [command usage](/cscli/cscli_ban_list/) for additional filtering and outpu
 > Flush all the existing bans
 
 ```bash
-{{cli.bin}} ban flush
+{{v0X.cli.bin}} ban flush
 ```
 
 !!! warning

docs/cheat_sheets/config-mgmt.md → docs/v0.3.X/docs/cheat_sheets/config-mgmt.md

@@ -1,4 +1,4 @@
-{{cli.bin}} allows you install, list, upgrade and remove configurations : parsers, enrichment, scenarios.
+{{v0X.cli.bin}} allows you install, list, upgrade and remove configurations : parsers, enrichment, scenarios.
 
 !!! warning
     If you're not running the latest CrowdSec version, configurations might not be the latest available. `cscli` will use the branch of the corresponding CrowdSec version to download and install configurations from the hub (it will use the `master` branch if you are on the latest CrowdSec version). 
@@ -9,26 +9,26 @@ _Parsers, Scenarios and Enrichers are often bundled together in "collections" to
 
 Parsers, scenarios, enrichers and collections all follow the same principle :
 
- - `{{cli.bin}} install parser crowdsec/nginx-logs`
- - `{{cli.bin}} update collection crowdsec/base-http-scenarios`
- - `{{cli.bin}} remove scenario crowdsec/mysql-bf`
+ - `{{v0X.cli.bin}} install parser crowdsec/nginx-logs`
+ - `{{v0X.cli.bin}} update collection crowdsec/base-http-scenarios`
+ - `{{v0X.cli.bin}} remove scenario crowdsec/mysql-bf`
 
-> Please see your local `{{cli.bin}} help` for up-to-date documentation
+> Please see your local `{{v0X.cli.bin}} help` for up-to-date documentation
 
 
 ## List configurations
 
 ```
-{{cli.bin}} list
+{{v0X.cli.bin}} list
 ```
 
 **note** `-a` allows for listing of uninstalled configurations as well
 
 <details>
-  <summary>{{cli.name}} list example</summary>
+  <summary>{{v0X.cli.name}} list example</summary>
 
 ```bash
-$ {{cli.bin}} list
+$ {{v0X.cli.bin}} list
 INFO[0000] Loaded 9 collecs, 14 parsers, 12 scenarios, 1 post-overflow parsers 
 INFO[0000] PARSERS:                                     
 --------------------------------------------------------------------------------------------------------------------
@@ -67,29 +67,29 @@ INFO[0000] POSTOVERFLOWS:
 
 
 
-For {{parsers.htmlname}}, {{scenarios.htmlname}}, {{collections.htmlname}} the outputs include, beside the version, the path and the name, a `STATUS` column :
+For {{v0X.parsers.htmlname}}, {{v0X.scenarios.htmlname}}, {{v0X.collections.htmlname}} the outputs include, beside the version, the path and the name, a `STATUS` column :
 
  - `✔️  enabled` : configuration is up-to-date
  - `⚠️  enabled,outdated` : a newer version is available
- - `🚫  enabled,local` : configuration is not managed by {{cli.name}}
+ - `🚫  enabled,local` : configuration is not managed by {{v0X.cli.name}}
  - `⚠️  enabled,tainted` : configuration has been locally modified
 
-(see `{{cli.name}} upgrade` to upgrade/sync your configurations with {{hub.htmlname}})
+(see `{{v0X.cli.name}} upgrade` to upgrade/sync your configurations with {{v0X.hub.htmlname}})
 
 ## Install new configurations
 
 
-`{{cli.bin}} install parser|scenario|postoverflow <name> [--force]`
+`{{v0X.cli.bin}} install parser|scenario|postoverflow <name> [--force]`
 
 
-  - `{{cli.bin}} install parser crowdsec/nginx-logs`
-  - `{{cli.bin}} install scenario crowdsec/http-scan-uniques_404`
+  - `{{v0X.cli.bin}} install parser crowdsec/nginx-logs`
+  - `{{v0X.cli.bin}} install scenario crowdsec/http-scan-uniques_404`
 
 
 ## Remove configurations
 
 
-`{{cli.bin}} remove parser|scenario|postoverflow <name> [--force]`
+`{{v0X.cli.bin}} remove parser|scenario|postoverflow <name> [--force]`
 
 
 ## Upgrade configurations
@@ -97,19 +97,19 @@ For {{parsers.htmlname}}, {{scenarios.htmlname}}, {{collections.htmlname}} the o
 > upgrade a specific scenario
 
 ```
-{{cli.bin}} upgrade scenario crowdsec/http-scan-uniques_404
+{{v0X.cli.bin}} upgrade scenario crowdsec/http-scan-uniques_404
 ```
 
 
 > upgrade **all** scenarios
 
 ```
-{{cli.bin}} upgrade scenario --all
+{{v0X.cli.bin}} upgrade scenario --all
 ```
 
 > upgrade **all** configurations (parsers, scenarios, collections, postoverflows)
 
 ```
-{{cli.bin}} upgrade --all
+{{v0X.cli.bin}} upgrade --all
 ```
 

docs/cheat_sheets/debugging_configs.md → docs/v0.3.X/docs/cheat_sheets/debugging_configs.md

@@ -25,12 +25,12 @@ WARN[05-08-2020 16:16:12] 182.x.x.x triggered a 4h0m0s ip ban remediation for [c
  - `-type` must respect expected log type (ie. `nginx` `syslog` etc.)
  - `-file` must point to a flat file or a gzip file
 
-When processing logs like this, {{crowdsec.name}} runs in "time machine" mode, and relies on the timestamps *in* the logs to evaluate scenarios. You will most likely need the `crowdsecurity/dateparse-enrich` parser for this.
+When processing logs like this, {{v0X.crowdsec.name}} runs in "time machine" mode, and relies on the timestamps *in* the logs to evaluate scenarios. You will most likely need the `crowdsecurity/dateparse-enrich` parser for this.
 
 
 ## Testing configurations on live system
 
-If you're playing around with parser/scenarios on a live system, you can use the `-t` (lint) option of {{crowdsec.Name}} to check your configurations validity before restarting/reloading services :
+If you're playing around with parser/scenarios on a live system, you can use the `-t` (lint) option of {{v0X.crowdsec.Name}} to check your configurations validity before restarting/reloading services :
 
 ```bash
 $ emacs /etc/crowdsec/config/scenarios/ssh-bf.yaml
@@ -91,7 +91,7 @@ DEBU[05-08-2020 16:02:26]        evt.Parsed.static_ressource = 'false'  cfg=blac
 
 # Test environments
 
-From a [{{crowdsec.name}} release archive]({{crowdsec.download_url}}), you can deploy a test (non-root) environment that is very suitable to write/debug/test parsers and scenarios. Environment is deployed using `./test_env.sh` script from tgz directory, and creates a test environment in `./tests` :
+From a [{{v0X.crowdsec.name}} release archive]({{v0X.crowdsec.download_url}}), you can deploy a test (non-root) environment that is very suitable to write/debug/test parsers and scenarios. Environment is deployed using `./test_env.sh` script from tgz directory, and creates a test environment in `./tests` :
 
 ```bash
 $ cd crowdsec-v0.3.0/

docs/contributing.md → docs/v0.3.X/docs/contributing.md

@@ -9,8 +9,8 @@ Help us improve the software and the user experience, to make the internet a saf
 
 If you spotted some mistakes in the documentation or have improvement suggestions, you can :
 
- - open a {{doc.new_issue}} if you are comfortable with github
- - let us know on {{doc.discourse}} if you want to discuss about it
+ - open a {{v0X.doc.new_issue}} if you are comfortable with github
+ - let us know on {{v0X.doc.discourse}} if you want to discuss about it
 
 Let us as well know if you have some improvement suggestions !
 
@@ -18,13 +18,13 @@ Let us as well know if you have some improvement suggestions !
 
 ## Contributing to the code
 
- - If you want to report a bug, you can use [the github bugtracker]({{crowdsec.bugreport}})
- - If you want to suggest an improvement you can use either [the github bugtracker]({{crowdsec.bugreport}}) or the {{doc.discourse}} if you want to discuss 
+ - If you want to report a bug, you can use [the github bugtracker]({{v0X.crowdsec.bugreport}})
+ - If you want to suggest an improvement you can use either [the github bugtracker]({{v0X.crowdsec.bugreport}}) or the {{v0X.doc.discourse}} if you want to discuss 
 
 
 ## Contributing to the parsers/scenarios
 
-If you want to contribute your parser or scenario to the community and have them appear on the {{hub.htmlname}}, you should [open a merge request](https://github.com/crowdsecurity/hub/pulls) on the hub.
+If you want to contribute your parser or scenario to the community and have them appear on the {{v0X.hub.htmlname}}, you should [open a merge request](https://github.com/crowdsecurity/hub/pulls) on the hub.
 
-We are currently working on a proper [CI](https://en.wikipedia.org/wiki/Continuous_integration) for the {{hub.htmlname}}, so for now all contribution are subject to peer-review, please bear with us !
+We are currently working on a proper [CI](https://en.wikipedia.org/wiki/Continuous_integration) for the {{v0X.hub.htmlname}}, so for now all contribution are subject to peer-review, please bear with us !
 

docs/cscli/cscli_api.md → docs/v0.3.X/docs/cscli/cscli_api.md

@@ -5,7 +5,7 @@ Crowdsec API interaction
 ### Synopsis
 
 
-Allow to register your machine into crowdsec API to send and receive alert.
+Allow to register your machine into crowdsec API to send and receive signal.
 		
 
 ### Examples