Merge branch 'master' into postoverflow_reinject_meta

.github/workflows/bats-hub.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest

.github/workflows/bats-mysql.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest

.github/workflows/bats-postgres.yml

@@ -10,14 +10,14 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
       database:
-        image: postgres:15
+        image: postgres:16
         env:
           POSTGRES_PASSWORD: "secret"
         ports:
@@ -30,13 +30,13 @@ jobs:
 
     steps:
 
-    - name: "Install pg_dump v15"
+    - name: "Install pg_dump v16"
       # we can remove this when it's released on ubuntu-latest
       run: |
           sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
           wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
           sudo apt update
-          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-15
+          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
 
     - name: "Force machineid"
       run: |

.github/workflows/bats-sqlite-coverage.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest

.github/workflows/bats.yml

@@ -31,7 +31,7 @@ jobs:
 
   # Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
   # CI, but they pass when run on devs' machines or in the release checks. We
-  # disable them here by default. Remove the if..false to enable them.
+  # disable them here by default. Remove if...false to enable them.
 
   mariadb:
     uses: ./.github/workflows/bats-mysql.yml

.github/workflows/ci-windows-build-msi.yml

@@ -23,7 +23,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: Build
     runs-on: windows-2019

.github/workflows/codeql-analysis.yml

@@ -45,6 +45,9 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
+      with:
+        # required to pick up tags for BUILD_VERSION
+        fetch-depth: 0
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -58,8 +61,8 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+    # - name: Autobuild
+    #   uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,9 +71,14 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+    - name: "Set up Go"
+      uses: actions/setup-go@v4
+      with:
+        go-version: "1.21.0"
+        cache-dependency-path: "**/go.sum"
+
+    - run: |
+       make clean build BUILD_RE2_WASM=1
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

.github/workflows/go-tests-windows.yml

@@ -22,7 +22,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: "Build + tests"
     runs-on: windows-2022

.github/workflows/go-tests.yml

@@ -34,7 +34,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -108,6 +108,18 @@ jobs:
           --health-timeout 10s
           --health-retries 5
 
+      loki:
+        image: grafana/loki:2.8.0
+        ports:
+          - "3100:3100"
+        options: >-
+          --name=loki1
+          --health-cmd "wget -q -O -  http://localhost:3100/ready | grep 'ready'"
+          --health-interval 30s
+          --health-timeout 10s
+          --health-retries 5
+          --health-start-period 30s
+
     steps:
 
     - name: Check out CrowdSec repository

.github/workflows/release_publish-package.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.8"]
+        go-version: ["1.21.4"]
 
     name: Build and upload binary package
     runs-on: ubuntu-latest
@@ -41,4 +41,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           tag_name="${GITHUB_REF##*/}"
-          hub release edit -a crowdsec-release.tgz -a vendor.tgz -a *-vendor.tar.xz -m "" "$tag_name"
+          gh release upload "$tag_name" crowdsec-release.tgz vendor.tgz *-vendor.tar.xz

.golangci.yml

@@ -199,6 +199,18 @@ issues:
         - govet
       text: "shadow: declaration of \"err\" shadows declaration"
 
+    #
+    # typecheck
+    #
+
+    - linters:
+        - typecheck
+      text: "undefined: min"
+
+    - linters:
+        - typecheck
+      text: "undefined: max"
+
     #
     # errcheck
     #

Dockerfile

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.20.8
+ARG GOVERSION=1.21.4
 
 FROM golang:${GOVERSION}-alpine AS build
 
@@ -32,7 +32,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
 
 FROM alpine:latest as slim
 
-RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
+RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash rsync && \
     mkdir -p /staging/etc/crowdsec && \
     mkdir -p /staging/etc/crowdsec/acquis.d && \
     mkdir -p /staging/var/lib/crowdsec && \

Dockerfile.debian

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.20.8
+ARG GOVERSION=1.21.4
 
 FROM golang:${GOVERSION}-bookworm AS build
 
@@ -47,7 +47,8 @@ RUN apt-get update && \
     iproute2 \
     ca-certificates \
     bash \
-    tzdata && \
+    tzdata \
+    rsync && \
     mkdir -p /staging/etc/crowdsec && \
     mkdir -p /staging/etc/crowdsec/acquis.d && \
     mkdir -p /staging/var/lib/crowdsec && \

Makefile

@@ -27,7 +27,7 @@ PLUGINS ?= $(patsubst ./cmd/notification-%,%,$(wildcard ./cmd/notification-*))
 
 # Can be overriden, if you can deal with the consequences
 BUILD_REQUIRE_GO_MAJOR ?= 1
-BUILD_REQUIRE_GO_MINOR ?= 20
+BUILD_REQUIRE_GO_MINOR ?= 21
 
 #--------------------------------------
 
@@ -165,8 +165,26 @@ plugins:
 		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
 	)
 
+# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
+.PHONY: clean-debian
+clean-debian:
+	@$(RM) -r debian/crowdsec
+	@$(RM) -r debian/crowdsec
+	@$(RM) -r debian/files
+	@$(RM) -r debian/.debhelper
+	@$(RM) -r debian/*.substvars
+	@$(RM) -r debian/*-stamp
+
+.PHONY: clean-rpm
+clean-rpm:
+	@$(RM) -r rpm/BUILD
+	@$(RM) -r rpm/BUILDROOT
+	@$(RM) -r rpm/RPMS
+	@$(RM) -r rpm/SOURCES/*.tar.gz
+	@$(RM) -r rpm/SRPMS
+
 .PHONY: clean
-clean: testclean
+clean: clean-debian clean-rpm testclean
 	@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
 	@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
 	@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)

azure-pipelines.yml

@@ -27,7 +27,7 @@ stages:
           - task: GoTool@0
             displayName: "Install Go 1.20"
             inputs:
-                version: '1.20.8'
+                version: '1.21.4'
 
           - pwsh: |
               choco install -y make

cmd/crowdsec-cli/bouncers.go

@@ -12,13 +12,12 @@ import (
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
+	"slices"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func getBouncers(out io.Writer, dbClient *database.Client) error {
@@ -26,16 +25,18 @@ func getBouncers(out io.Writer, dbClient *database.Client) error {
 	if err != nil {
 		return fmt.Errorf("unable to list bouncers: %s", err)
 	}
-	if csConfig.Cscli.Output == "human" {
+
+	switch csConfig.Cscli.Output {
+	case "human":
 		getBouncersTable(out, bouncers)
-	} else if csConfig.Cscli.Output == "json" {
+	case "json":
 		enc := json.NewEncoder(out)
 		enc.SetIndent("", "  ")
 		if err := enc.Encode(bouncers); err != nil {
 			return fmt.Errorf("failed to unmarshal: %w", err)
 		}
 		return nil
-	} else if csConfig.Cscli.Output == "raw" {
+	case "raw":
 		csvwriter := csv.NewWriter(out)
 		err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"})
 		if err != nil {
@@ -55,6 +56,7 @@ func getBouncers(out io.Writer, dbClient *database.Client) error {
 		}
 		csvwriter.Flush()
 	}
+
 	return nil
 }
 
@@ -78,12 +80,9 @@ func NewBouncersListCmd() *cobra.Command {
 }
 
 func runBouncersAdd(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
+	keyLength := 32
 
-	keyLength, err := flags.GetInt("length")
-	if err != nil {
-		return err
-	}
+	flags := cmd.Flags()
 
 	key, err := flags.GetString("key")
 	if err != nil {
@@ -108,13 +107,14 @@ func runBouncersAdd(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("unable to create bouncer: %s", err)
 	}
 
-	if csConfig.Cscli.Output == "human" {
+	switch csConfig.Cscli.Output {
+	case "human":
 		fmt.Printf("API key for '%s':\n\n", keyName)
 		fmt.Printf("   %s\n\n", apiKey)
 		fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
-	} else if csConfig.Cscli.Output == "raw" {
+	case "raw":
 		fmt.Printf("%s", apiKey)
-	} else if csConfig.Cscli.Output == "json" {
+	case "json":
 		j, err := json.Marshal(apiKey)
 		if err != nil {
 			return fmt.Errorf("unable to marshal api key")
@@ -127,19 +127,18 @@ func runBouncersAdd(cmd *cobra.Command, args []string) error {
 
 func NewBouncersAddCmd() *cobra.Command {
 	cmdBouncersAdd := &cobra.Command{
-		Use:   "add MyBouncerName [--length 16]",
+		Use:   "add MyBouncerName",
 		Short: "add a single bouncer to the database",
 		Example: `cscli bouncers add MyBouncerName
-cscli bouncers add MyBouncerName -l 24
-cscli bouncers add MyBouncerName -k <random-key>`,
+cscli bouncers add MyBouncerName --key <random-key>`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
 		RunE:              runBouncersAdd,
 	}
 
 	flags := cmdBouncersAdd.Flags()
-
-	flags.IntP("length", "l", 16, "length of the api key")
+	flags.StringP("length", "l", "", "length of the api key")
+	flags.MarkDeprecated("length", "use --key instead")
 	flags.StringP("key", "k", "", "api key for the bouncer")
 
 	return cmdBouncersAdd

cmd/crowdsec-cli/capi.go

@@ -60,16 +60,16 @@ func NewCapiRegisterCmd() *cobra.Command {
 		Short:             "Register to Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			var err error
 			capiUser, err := generateID(capiUserPrefix)
 			if err != nil {
-				log.Fatalf("unable to generate machine id: %s", err)
+				return fmt.Errorf("unable to generate machine id: %s", err)
 			}
 			password := strfmt.Password(generatePassword(passwordLength))
 			apiurl, err := url.Parse(types.CAPIBaseURL)
 			if err != nil {
-				log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
+				return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
 			}
 			_, err = apiclient.RegisterClient(&apiclient.Config{
 				MachineID:     capiUser,
@@ -80,7 +80,7 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}, nil)
 
 			if err != nil {
-				log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
+				return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
 			}
 			log.Printf("Successfully registered to Central API (CAPI)")
 
@@ -103,12 +103,12 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}
 			apiConfigDump, err := yaml.Marshal(apiCfg)
 			if err != nil {
-				log.Fatalf("unable to marshal api credentials: %s", err)
+				return fmt.Errorf("unable to marshal api credentials: %w", err)
 			}
 			if dumpFile != "" {
 				err = os.WriteFile(dumpFile, apiConfigDump, 0600)
 				if err != nil {
-					log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+					return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
 				}
 				log.Printf("Central API credentials dumped to '%s'", dumpFile)
 			} else {
@@ -116,6 +116,8 @@ func NewCapiRegisterCmd() *cobra.Command {
 			}
 
 			log.Warning(ReloadMessage())
+
+			return nil
 		},
 	}
 	cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
@@ -133,53 +135,57 @@ func NewCapiStatusCmd() *cobra.Command {
 		Short:             "Check status with the Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if csConfig.API.Server.OnlineClient == nil {
-				log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+				return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			}
 
 			if csConfig.API.Server.OnlineClient.Credentials == nil {
-				log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+				return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			}
 
 			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
+
 			apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
 			if err != nil {
-				log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
+				return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
 			}
 
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				return err
 			}
 
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to load hub index : %s", err)
-			}
-			scenarios, err := cwhub.GetInstalledScenariosAsString()
+			scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 			if err != nil {
-				log.Fatalf("failed to get scenarios : %s", err)
+				return fmt.Errorf("failed to get scenarios: %w", err)
 			}
+
 			if len(scenarios) == 0 {
-				log.Fatalf("no scenarios installed, abort")
+				return fmt.Errorf("no scenarios installed, abort")
 			}
 
 			Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
 			if err != nil {
-				log.Fatalf("init default client: %s", err)
+				return fmt.Errorf("init default client: %w", err)
 			}
+
 			t := models.WatcherAuthRequest{
 				MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
 				Password:  &password,
 				Scenarios: scenarios,
 			}
+
 			log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
+
 			_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
 			if err != nil {
-				log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
+				return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
 			}
 			log.Infof("You can successfully interact with Central API (CAPI)")
+
+			return nil
 		},
 	}
 

cmd/crowdsec-cli/collections.go

@@ -1,187 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCollectionsCmd() *cobra.Command {
-	var cmdCollections = &cobra.Command{
-		Use:   "collections [action]",
-		Short: "Manage collections from hub",
-		Long:  `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
-		/*TBD fix help*/
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"collection"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				return err
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("while setting hub branch: %w", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				return fmt.Errorf("failed to get hub index: %w", err)
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	var ignoreError bool
-
-	var cmdCollectionsInstall = &cobra.Command{
-		Use:     "install collection",
-		Short:   "Install given collection(s)",
-		Long:    `Fetch and install given collection(s) from hub`,
-		Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.COLLECTIONS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
-					Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
-	cmdCollections.AddCommand(cmdCollectionsInstall)
-
-	var cmdCollectionsRemove = &cobra.Command{
-		Use:               "remove collection",
-		Short:             "Remove given collection(s)",
-		Long:              `Remove given collection(s) from hub`,
-		Example:           `cscli collections remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one collection to remove or '--all'")
-			}
-
-			for _, name := range args {
-				if !forceAction {
-					item := cwhub.GetItem(cwhub.COLLECTIONS, name)
-					if item == nil {
-						return fmt.Errorf("unable to retrieve: %s", name)
-					}
-					if len(item.BelongsToCollections) > 0 {
-						log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
-						log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
-						continue
-					}
-				}
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
-			}
-			return nil
-		},
-	}
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
-	cmdCollections.AddCommand(cmdCollectionsRemove)
-
-	var cmdCollectionsUpgrade = &cobra.Command{
-		Use:               "upgrade collection",
-		Short:             "Upgrade given collection(s)",
-		Long:              `Fetch and upgrade given collection(s) from hub`,
-		Example:           `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one collection to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
-	cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-	cmdCollections.AddCommand(cmdCollectionsUpgrade)
-
-	var cmdCollectionsInspect = &cobra.Command{
-		Use:               "inspect collection",
-		Short:             "Inspect given collection",
-		Long:              `Inspect given collection`,
-		Example:           `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			for _, name := range args {
-				InspectItem(name, cwhub.COLLECTIONS)
-			}
-		},
-	}
-	cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-	cmdCollections.AddCommand(cmdCollectionsInspect)
-
-	var cmdCollectionsList = &cobra.Command{
-		Use:               "list collection [-a]",
-		Short:             "List all collections",
-		Long:              `List all collections`,
-		Example:           `cscli collections list`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
-		},
-	}
-	cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-	cmdCollections.AddCommand(cmdCollectionsList)
-
-	return cmdCollections
-}

cmd/crowdsec-cli/config_backup.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -9,9 +10,80 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-/* Backup crowdsec configurations to directory <dirPath> :
+func backupHub(dirPath string) error {
+	var itemDirectory string
+	var upstreamParsers []string
+
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, itemType := range cwhub.ItemTypes {
+		clog := log.WithFields(log.Fields{
+			"type": itemType,
+		})
+		itemMap := hub.GetItemMap(itemType)
+		if itemMap == nil {
+			clog.Infof("No %s to backup.", itemType)
+			continue
+		}
+		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
+		if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
+		}
+		upstreamParsers = []string{}
+		for k, v := range itemMap {
+			clog = clog.WithFields(log.Fields{
+				"file": v.Name,
+			})
+			if !v.State.Installed { //only backup installed ones
+				clog.Debugf("[%s] : not installed", k)
+				continue
+			}
+
+			//for the local/tainted ones, we back up the full file
+			if v.State.Tainted || v.IsLocal() || !v.State.UpToDate {
+				//we need to backup stages for parsers
+				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
+					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
+					if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
+					}
+				}
+				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate)
+				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
+				if err = CopyFile(v.State.LocalPath, tfile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
+				}
+				clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
+				continue
+			}
+			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
+			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
+			upstreamParsers = append(upstreamParsers, v.Name)
+		}
+		//write the upstream items
+		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
+		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
+		if err != nil {
+			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
+		}
+		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
+		if err != nil {
+			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
+		}
+		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
+	}
+
+	return nil
+}
+
+/*
+	Backup crowdsec configurations to directory <dirPath>:
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -19,6 +91,7 @@ import (
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
 func backupConfigToDirectory(dirPath string) error {
 	var err error
@@ -31,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
 	parentDir := filepath.Dir(dirPath)
-	if _, err := os.Stat(parentDir); err != nil {
+	if _, err = os.Stat(parentDir); err != nil {
 		return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
 	}
 
@@ -120,7 +193,7 @@ func backupConfigToDirectory(dirPath string) error {
 		log.Infof("Saved profiles to %s", backupProfiles)
 	}
 
-	if err = BackupHub(dirPath); err != nil {
+	if err = backupHub(dirPath); err != nil {
 		return fmt.Errorf("failed to backup hub config: %s", err)
 	}
 
@@ -128,15 +201,6 @@ func backupConfigToDirectory(dirPath string) error {
 }
 
 func runConfigBackup(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		return fmt.Errorf("failed to get Hub index: %w", err)
-	}
-
 	if err := backupConfigToDirectory(args[0]); err != nil {
 		return fmt.Errorf("failed to backup config: %w", err)
 	}

cmd/crowdsec-cli/config_feature_flags.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
@@ -87,7 +89,14 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
 
 	fmt.Println("To enable a feature you can: ")
 	fmt.Println("  - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
-	fmt.Printf("  - add the line '- <feature_name>' to the file %s/feature.yaml\n", csConfig.ConfigPaths.ConfigDir)
+
+	featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
+	if err != nil {
+		// we already read the file, shouldn't happen
+		return err
+	}
+
+	fmt.Printf("  - add the line '- <feature_name>' to the file %s\n", featurePath)
 	fmt.Println()
 
 	if len(enabled) == 0 && len(disabled) == 0 {

cmd/crowdsec-cli/config_restore.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
@@ -20,7 +21,94 @@ type OldAPICfg struct {
 	Password  string `json:"password"`
 }
 
-/* Restore crowdsec configurations to directory <dirPath> :
+func restoreHub(dirPath string) error {
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
+		return err
+	}
+
+	for _, itype := range cwhub.ItemTypes {
+		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
+		if _, err = os.Stat(itemDirectory); err != nil {
+			log.Infof("no %s in backup", itype)
+			continue
+		}
+		/*restore the upstream items*/
+		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
+		file, err := os.ReadFile(upstreamListFN)
+		if err != nil {
+			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
+		}
+		var upstreamList []string
+		err = json.Unmarshal(file, &upstreamList)
+		if err != nil {
+			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
+		}
+		for _, toinstall := range upstreamList {
+			item := hub.GetItem(itype, toinstall)
+			if item == nil {
+				log.Errorf("Item %s/%s not found in hub", itype, toinstall)
+				continue
+			}
+			err := item.Install(false, false)
+			if err != nil {
+				log.Errorf("Error while installing %s : %s", toinstall, err)
+			}
+		}
+
+		/*restore the local and tainted items*/
+		files, err := os.ReadDir(itemDirectory)
+		if err != nil {
+			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
+		}
+		for _, file := range files {
+			//this was the upstream data
+			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
+				continue
+			}
+			if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
+				//we expect a stage here
+				if !file.IsDir() {
+					continue
+				}
+				stage := file.Name()
+				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
+				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
+				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
+					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
+				}
+				/*find items*/
+				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
+				if err != nil {
+					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
+				}
+				//finally copy item
+				for _, tfile := range ifiles {
+					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
+					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
+					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
+					if err = CopyFile(sourceFile, destinationFile); err != nil {
+						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+					}
+					log.Infof("restored %s to %s", sourceFile, destinationFile)
+				}
+			} else {
+				log.Infof("Going to restore local/tainted [%s]", file.Name())
+				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
+				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
+				if err = CopyFile(sourceFile, destinationFile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+				}
+				log.Infof("restored %s to %s", sourceFile, destinationFile)
+			}
+
+		}
+	}
+	return nil
+}
+
+/*
+	Restore crowdsec configurations to directory <dirPath>:
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -28,6 +116,7 @@ type OldAPICfg struct {
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
 func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 	var err error
@@ -111,7 +200,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 
 	/*if there is a acquisition dir, restore its content*/
 	if csConfig.Crowdsec.AcquisitionDirPath != "" {
-		if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
+		if err = os.MkdirAll(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
 			return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
 		}
 	}
@@ -166,7 +255,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 		}
 	}
 
-	if err = RestoreHub(dirPath); err != nil {
+	if err = restoreHub(dirPath); err != nil {
 		return fmt.Errorf("failed to restore hub config : %s", err)
 	}
 
@@ -181,15 +270,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		return fmt.Errorf("failed to get Hub index: %w", err)
-	}
-
 	if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
 		return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
 	}

cmd/crowdsec-cli/config_show.go

@@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
 cscli:
   - Output                  : {{.Cscli.Output}}
   - Hub Branch              : {{.Cscli.HubBranch}}
-  - Hub Folder              : {{.Cscli.HubDir}}
 {{- end }}
 
 {{- if .API }}

cmd/crowdsec-cli/console.go

@@ -71,16 +71,12 @@ After running this command your will need to validate the enrollment in the weba
 				return fmt.Errorf("could not parse CAPI URL: %s", err)
 			}
 
-			if err := csConfig.LoadHub(); err != nil {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
 				return err
 			}
 
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				return fmt.Errorf("failed to load hub index: %s", err)
-			}
-
-			scenarios, err := cwhub.GetInstalledScenariosAsString()
+			scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 			if err != nil {
 				return fmt.Errorf("failed to get installed scenarios: %s", err)
 			}
@@ -234,6 +230,24 @@ Disable given information push to the central API.`,
 	return cmdConsole
 }
 
+func dumpConsoleConfig(c *csconfig.LocalApiServerCfg) error {
+	out, err := yaml.Marshal(c.ConsoleConfig)
+	if err != nil {
+		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
+	}
+
+	if c.ConsoleConfigPath == "" {
+		c.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
+		log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
+	}
+
+	if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
+		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
+	}
+
+	return nil
+}
+
 func SetConsoleOpts(args []string, wanted bool) error {
 	for _, arg := range args {
 		switch arg {
@@ -331,7 +345,7 @@ func SetConsoleOpts(args []string, wanted bool) error {
 		}
 	}
 
-	if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
+	if err := dumpConsoleConfig(csConfig.API.Server); err != nil {
 		return fmt.Errorf("failed writing console config: %s", err)
 	}
 

cmd/crowdsec-cli/dashboard.go

@@ -37,7 +37,7 @@ var (
 
 	forceYes bool
 
-	/*informations needed to setup a random password on user's behalf*/
+	// information needed to set up a random password on user's behalf
 )
 
 func NewDashboardCmd() *cobra.Command {

cmd/crowdsec-cli/hub.go

@@ -1,41 +1,30 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
 func NewHubCmd() *cobra.Command {
-	var cmdHub = &cobra.Command{
+	cmdHub := &cobra.Command{
 		Use:   "hub [action]",
-		Short: "Manage Hub",
-		Long: `
-Hub management
+		Short: "Manage hub index",
+		Long: `Hub management
 
 List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
-The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
-		`,
-		Example: `
-cscli hub list   # List all installed configurations
-cscli hub update # Download list of available configurations from the hub
-		`,
+The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
+		Example: `cscli hub list
+cscli hub update
+cscli hub upgrade`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			return nil
-		},
 	}
-	cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
 
 	cmdHub.AddCommand(NewHubListCmd())
 	cmdHub.AddCommand(NewHubUpdateCmd())
@@ -44,121 +33,142 @@ cscli hub update # Download list of available configurations from the hub
 	return cmdHub
 }
 
+func runHubList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, v := range hub.Warnings {
+		log.Info(v)
+	}
+
+	for _, line := range hub.ItemStats() {
+		log.Info(line)
+	}
+
+	items := make(map[string][]*cwhub.Item)
+
+	for _, itemType := range cwhub.ItemTypes {
+		items[itemType], err = selectItems(hub, itemType, nil, !all)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = listItems(color.Output, cwhub.ItemTypes, items)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func NewHubListCmd() *cobra.Command {
-	var cmdHubList = &cobra.Command{
+	cmdHubList := &cobra.Command{
 		Use:               "list [-a]",
-		Short:             "List installed configs",
+		Short:             "List all installed configurations",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			//use LocalSync to get warnings about tainted / outdated items
-			_, warn := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
-			}
-			cwhub.DisplaySummary()
-			ListItems(color.Output, []string{
-				cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
-			}, args, true, false, all)
-		},
+		RunE:              runHubList,
 	}
-	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+
+	flags := cmdHubList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
 
 	return cmdHubList
 }
 
+func runHubUpdate(cmd *cobra.Command, args []string) error {
+	local := csConfig.Hub
+	remote := require.RemoteHub(csConfig)
+
+	// don't use require.Hub because if there is no index file, it would fail
+	hub, err := cwhub.NewHub(local, remote, true)
+	if err != nil {
+		return fmt.Errorf("failed to update hub: %w", err)
+	}
+
+	for _, v := range hub.Warnings {
+		log.Info(v)
+	}
+
+	return nil
+}
+
 func NewHubUpdateCmd() *cobra.Command {
-	var cmdHubUpdate = &cobra.Command{
+	cmdHubUpdate := &cobra.Command{
 		Use:   "update",
-		Short: "Fetch available configs from hub",
+		Short: "Download the latest index (catalog of available configurations)",
 		Long: `
-Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
+Fetches the .index.json file from the hub, containing the list of available configs.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
+		RunE:              runHubUpdate,
+	}
 
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-			return nil
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-				if errors.Is(err, cwhub.ErrIndexNotFound) {
-					log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
-					cwhub.HubBranch = "master"
-					if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-						log.Fatalf("Failed to get Hub index after retry : %v", err)
-					}
-				} else {
-					log.Fatalf("Failed to get Hub index : %v", err)
-				}
+	return cmdHubUpdate
+}
+
+func runHubUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
+		return err
+	}
+
+	for _, itemType := range cwhub.ItemTypes {
+		items, err := hub.GetInstalledItems(itemType)
+		if err != nil {
+			return err
+		}
+
+		updated := 0
+
+		log.Infof("Upgrading %s", itemType)
+		for _, item := range items {
+			didUpdate, err := item.Upgrade(force)
+			if err != nil {
+				return err
 			}
-			//use LocalSync to get warnings about tainted / outdated items
-			_, warn := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
+			if didUpdate {
+				updated++
 			}
-		},
+		}
+		log.Infof("Upgraded %d %s", updated, itemType)
 	}
 
-	return cmdHubUpdate
+	return nil
 }
 
 func NewHubUpgradeCmd() *cobra.Command {
-	var cmdHubUpgrade = &cobra.Command{
+	cmdHubUpgrade := &cobra.Command{
 		Use:   "upgrade",
-		Short: "Upgrade all configs installed from hub",
+		Short: "Upgrade all configurations to their latest version",
 		Long: `
 Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-			return nil
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-
-			log.Infof("Upgrading collections")
-			cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			log.Infof("Upgrading parsers")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			log.Infof("Upgrading scenarios")
-			cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			log.Infof("Upgrading postoverflows")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-		},
+		RunE:              runHubUpgrade,
 	}
-	cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	flags := cmdHubUpgrade.Flags()
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
 
 	return cmdHubUpgrade
 }

cmd/crowdsec-cli/hubtest.go

@@ -18,9 +18,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
-var (
-	HubTest hubtest.HubTest
-)
+var HubTest hubtest.HubTest
 
 func NewHubTestCmd() *cobra.Command {
 	var hubPath string
@@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command {
 			return nil
 		},
 	}
+
 	cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
 	cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
 	cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
@@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command {
 	return cmdHubTest
 }
 
-
 func NewHubTestCreateCmd() *cobra.Command {
 	parsers := []string{}
 	postoverflows := []string{}
@@ -138,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			}
 
 			configFilePath := filepath.Join(testPath, "config.yaml")
-			fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+			fd, err := os.Create(configFilePath)
 			if err != nil {
 				return fmt.Errorf("open: %s", err)
 			}
@@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			return nil
 		},
 	}
+
 	cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
 	cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
 	cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
@@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 	return cmdHubTestCreate
 }
 
-
 func NewHubTestRunCmd() *cobra.Command {
 	var noClean bool
 	var runAll bool
@@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if !runAll && len(args) == 0 {
 				printHelp(cmd)
-				return fmt.Errorf("Please provide test to run or --all flag")
+				return fmt.Errorf("please provide test to run or --all flag")
 			}
 
 			if runAll {
@@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command {
 				}
 			}
 
+			// set timezone to avoid DST issues
+			os.Setenv("TZ", "UTC")
+
 			for _, test := range HubTest.Tests {
 				if csConfig.Cscli.Output == "human" {
 					log.Infof("Running test '%s'", test.Name)
@@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command {
 					}
 				}
 			}
-			if csConfig.Cscli.Output == "human" {
+
+			switch csConfig.Cscli.Output {
+			case "human":
 				hubTestResultTable(color.Output, testResult)
-			} else if csConfig.Cscli.Output == "json" {
+			case "json":
 				jsonResult := make(map[string][]string, 0)
 				jsonResult["success"] = make([]string, 0)
 				jsonResult["fail"] = make([]string, 0)
@@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command {
 					return fmt.Errorf("unable to json test result: %s", err)
 				}
 				fmt.Println(string(jsonStr))
+			default:
+				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			if !success {
@@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command {
 			return nil
 		},
 	}
+
 	cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
 	cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
 	cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
@@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command {
 	return cmdHubTestRun
 }
 
-
 func NewHubTestCleanCmd() *cobra.Command {
 	var cmdHubTestClean = &cobra.Command{
 		Use:               "clean",
@@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command {
 	return cmdHubTestClean
 }
 
-
 func NewHubTestInfoCmd() *cobra.Command {
 	var cmdHubTestInfo = &cobra.Command{
 		Use:               "info",
@@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command {
 	return cmdHubTestInfo
 }
 
-
 func NewHubTestListCmd() *cobra.Command {
 	var cmdHubTestList = &cobra.Command{
 		Use:               "list",
@@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command {
 	return cmdHubTestList
 }
 
-
 func NewHubTestCoverageCmd() *cobra.Command {
 	var showParserCov bool
 	var showScenarioCov bool
@@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				return fmt.Errorf("unable to load all tests: %+v", err)
 			}
 			var err error
-			scenarioCoverage := []hubtest.ScenarioCoverage{}
-			parserCoverage := []hubtest.ParserCoverage{}
+			scenarioCoverage := []hubtest.Coverage{}
+			parserCoverage := []hubtest.Coverage{}
 			scenarioCoveragePercent := 0
 			parserCoveragePercent := 0
 
@@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				parserTested := 0
 				for _, test := range parserCoverage {
 					if test.TestsCount > 0 {
-						parserTested += 1
+						parserTested++
 					}
 				}
 				parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("while getting scenario coverage: %s", err)
 				}
+
 				scenarioTested := 0
 				for _, test := range scenarioCoverage {
 					if test.TestsCount > 0 {
-						scenarioTested += 1
+						scenarioTested++
 					}
 				}
+
 				scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
 			}
 
@@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				os.Exit(0)
 			}
 
-			if csConfig.Cscli.Output == "human" {
+			switch csConfig.Cscli.Output {
+			case "human":
 				if showParserCov || showAll {
 					hubTestParserCoverageTable(color.Output, parserCoverage)
 				}
@@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					fmt.Printf("SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
 				}
-			} else if csConfig.Cscli.Output == "json" {
+			case "json":
 				dump, err := json.MarshalIndent(parserCoverage, "", " ")
 				if err != nil {
 					return err
@@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
 					return err
 				}
 				fmt.Printf("%s", dump)
-			} else {
+			default:
 				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			return nil
 		},
 	}
+
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
@@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command {
 	return cmdHubTestCoverage
 }
 
-
 func NewHubTestEvalCmd() *cobra.Command {
 	var evalExpression string
 	var cmdHubTestEval = &cobra.Command{
@@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("can't load test: %+v", err)
 				}
+
 				err = test.ParserAssert.LoadTest(test.ParserResultFile)
 				if err != nil {
 					return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
 				}
+
 				output, err := test.ParserAssert.EvalExpression(evalExpression)
 				if err != nil {
 					return err
 				}
+
 				fmt.Print(output)
 			}
 
 			return nil
 		},
 	}
+
 	cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
 
 	return cmdHubTestEval
 }
 
-
 func NewHubTestExplainCmd() *cobra.Command {
 	var cmdHubTestExplain = &cobra.Command{
 		Use:               "explain",
@@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command {
 				}
 				err = test.ParserAssert.LoadTest(test.ParserResultFile)
 				if err != nil {
-					err := test.Run()
-					if err != nil {
+					if err = test.Run(); err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-					err = test.ParserAssert.LoadTest(test.ParserResultFile)
-					if err != nil {
+
+					if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
 						return fmt.Errorf("unable to load parser result after run: %s", err)
 					}
 				}
 
 				err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
 				if err != nil {
-					err := test.Run()
-					if err != nil {
+					if err = test.Run(); err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-					err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
-					if err != nil {
+
+					if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
 						return fmt.Errorf("unable to load scenario result after run: %s", err)
 					}
 				}

cmd/crowdsec-cli/hubtest_table.go

@@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
 	t.Render()
 }
 
-func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
+func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Parser", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
+
 	for _, test := range coverage {
 		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
 			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-		t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()
 }
 
-func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
+func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Scenario", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
+
 	for _, test := range coverage {
 		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
 			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-		t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()

cmd/crowdsec-cli/item_metrics.go

@@ -0,0 +1,236 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/trace"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func ShowMetrics(hubItem *cwhub.Item) error {
+	switch hubItem.Type {
+	case cwhub.PARSERS:
+		metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		parserMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.SCENARIOS:
+		metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.COLLECTIONS:
+		for _, sub := range hubItem.SubItems() {
+			if err := ShowMetrics(sub); err != nil {
+				return err
+			}
+		}
+	default:
+		// no metrics for this item type
+	}
+	return nil
+}
+
+// GetParserMetric is a complete rip from prom2json
+func GetParserMetric(url string, itemName string) map[string]map[string]int {
+	stats := make(map[string]map[string]int)
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric %v", metric.Labels)
+			} else {
+				if srctype, ok := metric.Labels["type"]; ok {
+					source = srctype + ":" + source
+				}
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_reader_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+					stats[source]["parsed"] = 0
+					stats[source]["reads"] = 0
+					stats[source]["unparsed"] = 0
+					stats[source]["hits"] = 0
+				}
+				stats[source]["reads"] += ival
+			case "cs_parser_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			case "cs_node_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["hits"] += ival
+			case "cs_node_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_node_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetScenarioMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["instantiation"] = 0
+	stats["curr_count"] = 0
+	stats["overflow"] = 0
+	stats["pour"] = 0
+	stats["underflow"] = 0
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_bucket_created_total":
+				stats["instantiation"] += ival
+			case "cs_buckets":
+				stats["curr_count"] += ival
+			case "cs_bucket_overflowed_total":
+				stats["overflow"] += ival
+			case "cs_bucket_poured_total":
+				stats["pour"] += ival
+			case "cs_bucket_underflowed_total":
+				stats["underflow"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetPrometheusMetric(url string) []*prom2json.Family {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+
+	return result
+}
+
+type unit struct {
+	value  int64
+	symbol string
+}
+
+var ranges = []unit{
+	{value: 1e18, symbol: "E"},
+	{value: 1e15, symbol: "P"},
+	{value: 1e12, symbol: "T"},
+	{value: 1e9, symbol: "G"},
+	{value: 1e6, symbol: "M"},
+	{value: 1e3, symbol: "k"},
+	{value: 1, symbol: ""},
+}
+
+func formatNumber(num int) string {
+	goodUnit := unit{}
+	for _, u := range ranges {
+		if int64(num) >= u.value {
+			goodUnit = u
+			break
+		}
+	}
+
+	if goodUnit.value == 1 {
+		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
+	}
+
+	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
+	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
+}

cmd/crowdsec-cli/item_suggest.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/agext/levenshtein"
+	"github.com/spf13/cobra"
+	"slices"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+const MaxDistance = 7
+
+// SuggestNearestMessage returns a message with the most similar item name, if one is found
+func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
+	score := 100
+	nearest := ""
+
+	for _, item := range hub.GetItemMap(itemType) {
+		d := levenshtein.Distance(itemName, item.Name, nil)
+		if d < score {
+			score = d
+			nearest = item.Name
+		}
+	}
+
+	msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
+
+	if score < MaxDistance {
+		msg += fmt.Sprintf(", did you mean '%s'?", nearest)
+	}
+
+	return msg
+}
+
+func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	for _, item := range hub.GetItemMap(itemType) {
+		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
+			comp = append(comp, item.Name)
+		}
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}
+
+func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	items, err := hub.GetInstalledItemNames(itemType)
+	if err != nil {
+		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	if toComplete != "" {
+		for _, item := range items {
+			if strings.Contains(item, toComplete) {
+				comp = append(comp, item)
+			}
+		}
+	} else {
+		comp = items
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}

cmd/crowdsec-cli/itemcommands.go

@@ -0,0 +1,606 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/go-cs-lib/coalesce"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+type cmdHelp struct {
+	// Example is required, the others have a default value
+	// generated from the item type
+	use     string
+	short   string
+	long    string
+	example string
+}
+
+type hubItemType struct {
+	name        string // plural, as used in the hub index
+	singular    string
+	oneOrMore   string // parenthetical pluralizaion: "parser(s)"
+	help        cmdHelp
+	installHelp cmdHelp
+	removeHelp  cmdHelp
+	upgradeHelp cmdHelp
+	inspectHelp cmdHelp
+	listHelp    cmdHelp
+}
+
+var hubItemTypes = map[string]hubItemType{
+	"parsers": {
+		name:      "parsers",
+		singular:  "parser",
+		oneOrMore: "parser(s)",
+		help: cmdHelp{
+			example: `cscli parsers list -a
+cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli parsers list
+cscli parsers list -a
+cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+
+List only enabled parsers unless "-a" or names are specified.`,
+		},
+	},
+	"postoverflows": {
+		name:      "postoverflows",
+		singular:  "postoverflow",
+		oneOrMore: "postoverflow(s)",
+		help: cmdHelp{
+			example: `cscli postoverflows list -a
+cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli postoverflows list
+cscli postoverflows list -a
+cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
+
+List only enabled postoverflows unless "-a" or names are specified.`,
+		},
+	},
+	"scenarios": {
+		name:      "scenarios",
+		singular:  "scenario",
+		oneOrMore: "scenario(s)",
+		help: cmdHelp{
+			example: `cscli scenarios list -a
+cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli scenarios list
+cscli scenarios list -a
+cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
+
+List only enabled scenarios unless "-a" or names are specified.`,
+		},
+	},
+	"collections": {
+		name:      "collections",
+		singular:  "collection",
+		oneOrMore: "collection(s)",
+		help: cmdHelp{
+			example: `cscli collections list -a
+cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli collections list
+cscli collections list -a
+cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
+
+List only enabled collections unless "-a" or names are specified.`,
+		},
+	},
+}
+
+func NewItemsCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
+		Short:             coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
+		Long:              it.help.long,
+		Example:           it.help.example,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{it.singular},
+		DisableAutoGenTag: true,
+	}
+
+	cmd.AddCommand(NewItemsInstallCmd(typeName))
+	cmd.AddCommand(NewItemsRemoveCmd(typeName))
+	cmd.AddCommand(NewItemsUpgradeCmd(typeName))
+	cmd.AddCommand(NewItemsInspectCmd(typeName))
+	cmd.AddCommand(NewItemsListCmd(typeName))
+
+	return cmd
+}
+
+func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		downloadOnly, err := flags.GetBool("download-only")
+		if err != nil {
+			return err
+		}
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		ignoreError, err := flags.GetBool("ignore")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+		if err != nil {
+			return err
+		}
+
+		for _, name := range args {
+			item := hub.GetItem(it.name, name)
+			if item == nil {
+				msg := SuggestNearestMessage(hub, it.name, name)
+				if !ignoreError {
+					return fmt.Errorf(msg)
+				}
+
+				log.Errorf(msg)
+
+				continue
+			}
+
+			if err := item.Install(force, downloadOnly); err != nil {
+				if !ignoreError {
+					return fmt.Errorf("error while installing '%s': %w", item.Name, err)
+				}
+				log.Errorf("Error while installing '%s': %s", item.Name, err)
+			}
+		}
+
+		log.Infof(ReloadMessage())
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsInstallCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.installHelp.use, "install [item]..."),
+		Short:             coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
+		Example:           it.installHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(typeName, args, toComplete)
+		},
+		RunE: itemsInstallRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
+
+	return cmd
+}
+
+// return the names of the installed parents of an item, used to check if we can remove it
+func istalledParentNames(item *cwhub.Item) []string {
+	ret := make([]string, 0)
+
+	for _, parent := range item.Ancestors() {
+		if parent.State.Installed {
+			ret = append(ret, parent.Name)
+		}
+	}
+
+	return ret
+}
+
+func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		purge, err := flags.GetBool("purge")
+		if err != nil {
+			return err
+		}
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, nil)
+		if err != nil {
+			return err
+		}
+
+		if all {
+			getter := hub.GetInstalledItems
+			if purge {
+				getter = hub.GetAllItems
+			}
+
+			items, err := getter(it.name)
+			if err != nil {
+				return err
+			}
+
+			removed := 0
+
+			for _, item := range items {
+				didRemove, err := item.Remove(purge, force)
+				if err != nil {
+					return err
+				}
+				if didRemove {
+					removed++
+				}
+			}
+
+			log.Infof("Removed %d %s", removed, it.name)
+			if removed > 0 {
+				log.Infof(ReloadMessage())
+			}
+
+			return nil
+		}
+
+		if len(args) == 0 {
+			return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
+		}
+
+		removed := 0
+
+		for _, itemName := range args {
+			item := hub.GetItem(it.name, itemName)
+			if item == nil {
+				return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
+			}
+
+			parents := istalledParentNames(item)
+
+			if !force && len(parents) > 0 {
+				log.Warningf("%s belongs to collections: %s", item.Name, parents)
+				log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
+				continue
+			}
+
+			didRemove, err := item.Remove(purge, force)
+			if err != nil {
+				return err
+			}
+
+			if didRemove {
+				log.Infof("Removed %s", item.Name)
+				removed++
+			}
+		}
+		if removed > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+	return run
+}
+
+func NewItemsRemoveCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.removeHelp.use, "remove [item]..."),
+		Short:             coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
+		Example:           it.removeHelp.example,
+		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsRemoveRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
+
+	return cmd
+}
+
+func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+		if err != nil {
+			return err
+		}
+
+		if all {
+			items, err := hub.GetInstalledItems(it.name)
+			if err != nil {
+				return err
+			}
+
+			updated := 0
+
+			for _, item := range items {
+				didUpdate, err := item.Upgrade(force)
+				if err != nil {
+					return err
+				}
+				if didUpdate {
+					updated++
+				}
+			}
+
+			log.Infof("Updated %d %s", updated, it.name)
+
+			if updated > 0 {
+				log.Infof(ReloadMessage())
+			}
+
+			return nil
+		}
+
+		if len(args) == 0 {
+			return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
+		}
+
+		updated := 0
+
+		for _, itemName := range args {
+			item := hub.GetItem(it.name, itemName)
+			if item == nil {
+				return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
+			}
+
+			didUpdate, err := item.Upgrade(force)
+			if err != nil {
+				return err
+			}
+
+			if didUpdate {
+				log.Infof("Updated %s", item.Name)
+				updated++
+			}
+		}
+		if updated > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsUpgradeCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
+		Short:             coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
+		Example:           it.upgradeHelp.example,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsUpgradeRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmd
+}
+
+func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		url, err := flags.GetString("url")
+		if err != nil {
+			return err
+		}
+
+		if url != "" {
+			csConfig.Cscli.PrometheusUrl = url
+		}
+
+		noMetrics, err := flags.GetBool("no-metrics")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, name := range args {
+			item := hub.GetItem(it.name, name)
+			if item == nil {
+				return fmt.Errorf("can't find '%s' in %s", name, it.name)
+			}
+			if err = InspectItem(item, !noMetrics); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsInspectCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.inspectHelp.use, "inspect [item]..."),
+		Short:             coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
+		Example:           it.inspectHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsInspectRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
+
+	return cmd
+}
+
+func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, nil)
+		if err != nil {
+			return err
+		}
+
+		items := make(map[string][]*cwhub.Item)
+
+		items[it.name], err = selectItems(hub, it.name, args, !all)
+		if err != nil {
+			return err
+		}
+
+		if err = listItems(color.Output, []string{it.name}, items); err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsListCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.listHelp.use, "list [item... | -a]"),
+		Short:             coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
+		Long:              coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
+		Example:           it.listHelp.example,
+		DisableAutoGenTag: true,
+		RunE:              itemsListRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmd
+}

cmd/crowdsec-cli/items.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+	"slices"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
+func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
+	itemNames := hub.GetItemNames(itemType)
+
+	notExist := []string{}
+
+	if len(args) > 0 {
+		for _, arg := range args {
+			if !slices.Contains(itemNames, arg) {
+				notExist = append(notExist, arg)
+			}
+		}
+	}
+
+	if len(notExist) > 0 {
+		return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
+	}
+
+	if len(args) > 0 {
+		itemNames = args
+		installedOnly = false
+	}
+
+	items := make([]*cwhub.Item, 0, len(itemNames))
+
+	for _, itemName := range itemNames {
+		item := hub.GetItem(itemType, itemName)
+		if installedOnly && !item.State.Installed {
+			continue
+		}
+
+		items = append(items, item)
+	}
+
+	cwhub.SortItemSlice(items)
+
+	return items, nil
+}
+
+func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
+	switch csConfig.Cscli.Output {
+	case "human":
+		for _, itemType := range itemTypes {
+			listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
+		}
+	case "json":
+		type itemHubStatus struct {
+			Name         string `json:"name"`
+			LocalVersion string `json:"local_version"`
+			LocalPath    string `json:"local_path"`
+			Description  string `json:"description"`
+			UTF8Status   string `json:"utf8_status"`
+			Status       string `json:"status"`
+		}
+
+		hubStatus := make(map[string][]itemHubStatus)
+		for _, itemType := range itemTypes {
+			// empty slice in case there are no items of this type
+			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
+
+			for i, item := range items[itemType] {
+				status, emo := item.InstallStatus()
+				hubStatus[itemType][i] = itemHubStatus{
+					Name:         item.Name,
+					LocalVersion: item.State.LocalVersion,
+					LocalPath:    item.State.LocalPath,
+					Description:  item.Description,
+					Status:       status,
+					UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
+				}
+			}
+		}
+
+		x, err := json.MarshalIndent(hubStatus, "", " ")
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal: %w", err)
+		}
+
+		out.Write(x)
+	case "raw":
+		csvwriter := csv.NewWriter(out)
+
+		header := []string{"name", "status", "version", "description"}
+		if len(itemTypes) > 1 {
+			header = append(header, "type")
+		}
+
+		if err := csvwriter.Write(header); err != nil {
+			return fmt.Errorf("failed to write header: %s", err)
+		}
+
+		for _, itemType := range itemTypes {
+			for _, item := range items[itemType] {
+				status, _ := item.InstallStatus()
+				row := []string{
+					item.Name,
+					status,
+					item.State.LocalVersion,
+					item.Description,
+				}
+				if len(itemTypes) > 1 {
+					row = append(row, itemType)
+				}
+				if err := csvwriter.Write(row); err != nil {
+					return fmt.Errorf("failed to write raw output: %s", err)
+				}
+			}
+		}
+		csvwriter.Flush()
+	default:
+		return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
+	}
+
+	return nil
+}
+
+func InspectItem(item *cwhub.Item, showMetrics bool) error {
+	switch csConfig.Cscli.Output {
+	case "human", "raw":
+		enc := yaml.NewEncoder(os.Stdout)
+		enc.SetIndent(2)
+		if err := enc.Encode(item); err != nil {
+			return fmt.Errorf("unable to encode item: %s", err)
+		}
+	case "json":
+		b, err := json.MarshalIndent(*item, "", "  ")
+		if err != nil {
+			return fmt.Errorf("unable to marshal item: %s", err)
+		}
+		fmt.Print(string(b))
+	}
+
+	if csConfig.Cscli.Output == "human" && showMetrics {
+		fmt.Printf("\nCurrent metrics: \n")
+		if err := ShowMetrics(item); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/crowdsec-cli/lapi.go

@@ -5,17 +5,18 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"slices"
 	"sort"
 	"strings"
 
 	"github.com/go-openapi/strfmt"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -36,15 +37,13 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
 	}
-	if err := csConfig.LoadHub(); err != nil {
+
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
 		log.Fatal(err)
 	}
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		log.Fatalf("Failed to load hub index : %s", err)
-	}
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		log.Fatalf("failed to get scenarios : %s", err)
 	}
@@ -340,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs
 				log.Fatalf("Failed to init expr helpers : %s", err)
 			}
 
-			// Populate cwhub package tools
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Fatalf("Failed to load hub index : %s", err)
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				log.Fatal(err)
 			}
 
-			csParsers := parser.NewParsers()
+			csParsers := parser.NewParsers(hub)
 			if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
 				log.Fatalf("unable to load parsers: %s", err)
 			}

cmd/crowdsec-cli/machines.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"math/big"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
@@ -17,7 +18,6 @@ import (
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/machineid"

cmd/crowdsec-cli/main.go

@@ -11,10 +11,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
-	"golang.org/x/exp/slices"
+	"slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
@@ -29,15 +28,11 @@ var dbClient *database.Client
 var OutputFormat string
 var OutputColor string
 
-var downloadOnly bool
-var forceAction bool
-var purge bool
-var all bool
-
-var prometheusURL string
-
 var mergedConfig string
 
+// flagBranch overrides the value in csConfig.Cscli.HubBranch
+var flagBranch = ""
+
 func initConfig() {
 	var err error
 	if trace_lvl {
@@ -58,9 +53,6 @@ func initConfig() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		if err := csConfig.LoadCSCLI(); err != nil {
-			log.Fatal(err)
-		}
 	} else {
 		csConfig = csconfig.NewDefaultConfig()
 	}
@@ -71,13 +63,10 @@ func initConfig() {
 		log.Debugf("Enabled feature flags: %s", fflist)
 	}
 
-	if csConfig.Cscli == nil {
-		log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
+	if flagBranch != "" {
+		csConfig.Cscli.HubBranch = flagBranch
 	}
 
-	if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
-		cwhub.HubBranch = csConfig.Cscli.HubBranch
-	}
 	if OutputFormat != "" {
 		csConfig.Cscli.Output = OutputFormat
 		if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
@@ -134,7 +123,7 @@ var (
 
 func main() {
 	// set the formatter asap and worry about level later
-	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
+	logFormatter := &log.TextFormatter{TimestampFormat: "2006-01-02 15:04:05", FullTimestamp: true}
 	log.SetFormatter(logFormatter)
 
 	if err := fflag.RegisterAllFeatures(); err != nil {
@@ -206,7 +195,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
 	rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
 
-	rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
+	rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
 	if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
 		log.Fatalf("failed to hide flag: %s", err)
 	}
@@ -243,10 +232,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewSimulationCmds())
 	rootCmd.AddCommand(NewBouncersCmd())
 	rootCmd.AddCommand(NewMachinesCmd())
-	rootCmd.AddCommand(NewParsersCmd())
-	rootCmd.AddCommand(NewScenariosCmd())
-	rootCmd.AddCommand(NewCollectionsCmd())
-	rootCmd.AddCommand(NewPostOverflowsCmd())
 	rootCmd.AddCommand(NewCapiCmd())
 	rootCmd.AddCommand(NewLapiCmd())
 	rootCmd.AddCommand(NewCompletionCmd())
@@ -255,6 +240,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewHubTestCmd())
 	rootCmd.AddCommand(NewNotificationsCmd())
 	rootCmd.AddCommand(NewSupportCmd())
+	rootCmd.AddCommand(NewItemsCmd("collections"))
+	rootCmd.AddCommand(NewItemsCmd("parsers"))
+	rootCmd.AddCommand(NewItemsCmd("scenarios"))
+	rootCmd.AddCommand(NewItemsCmd("postoverflows"))
 
 	if fflag.CscliSetup.IsEnabled() {
 		rootCmd.AddCommand(NewSetupCmd())

cmd/crowdsec-cli/metrics.go

@@ -284,29 +284,32 @@ var noUnit bool
 
 
 func runMetrics(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadPrometheus(); err != nil {
-		return fmt.Errorf("failed to load prometheus config: %w", err)
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
 	}
 
-	if csConfig.Prometheus == nil {
-		return fmt.Errorf("prometheus section missing, can't show metrics")
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
 	}
 
-	if !csConfig.Prometheus.Enabled {
-		return fmt.Errorf("prometheus is not enabled, can't show metrics")
+	noUnit, err = flags.GetBool("no-unit")
+	if err != nil {
+		return err
 	}
 
-	if prometheusURL == "" {
-		prometheusURL = csConfig.Cscli.PrometheusUrl
+	if csConfig.Prometheus == nil {
+		return fmt.Errorf("prometheus section missing, can't show metrics")
 	}
 
-	if prometheusURL == "" {
-		return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
+	if !csConfig.Prometheus.Enabled {
+		return fmt.Errorf("prometheus is not enabled, can't show metrics")
 	}
 
-	err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
-	if err != nil {
-		return fmt.Errorf("could not fetch prometheus metrics: %w", err)
+	if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
+		return err
 	}
 	return nil
 }
@@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: runMetrics,
 	}
-	cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
-	cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
+
+	flags := cmdMetrics.PersistentFlags()
+	flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
+	flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
 
 	return cmdMetrics
 }

cmd/crowdsec-cli/notifications.go

@@ -18,15 +18,19 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/tomb.v2"
+	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
 )
 
 type NotificationsCfg struct {
@@ -61,11 +65,12 @@ func NewNotificationsCmd() *cobra.Command {
 	cmdNotifications.AddCommand(NewNotificationsListCmd())
 	cmdNotifications.AddCommand(NewNotificationsInspectCmd())
 	cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
+	cmdNotifications.AddCommand(NewNotificationsTestCmd())
 
 	return cmdNotifications
 }
 
-func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
+func getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
 	pcfgs := map[string]csplugin.PluginConfig{}
 	wf := func(path string, info fs.FileInfo, err error) error {
 		if info == nil {
@@ -78,6 +83,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
 				return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
 			}
 			for _, t := range ts {
+				csplugin.SetRequiredFields(&t)
 				pcfgs[t.Name] = t
 			}
 		}
@@ -87,8 +93,15 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
 	if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
 		return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
 	}
+	return pcfgs, nil
+}
 
+func getProfilesConfigs() (map[string]NotificationsCfg, error) {
 	// A bit of a tricky stuf now: reconcile profiles and notification plugins
+	pcfgs, err := getPluginConfigs()
+	if err != nil {
+		return nil, err
+	}
 	ncfgs := map[string]NotificationsCfg{}
 	profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
 	if err != nil {
@@ -131,13 +144,13 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
 func NewNotificationsListCmd() *cobra.Command {
 	var cmdNotificationsList = &cobra.Command{
 		Use:               "list",
-		Short:             "List active notifications plugins",
-		Long:              `List active notifications plugins`,
+		Short:             "list active notifications plugins",
+		Long:              `list active notifications plugins`,
 		Example:           `cscli notifications list`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, arg []string) error {
-			ncfgs, err := getNotificationsConfiguration()
+			ncfgs, err := getProfilesConfigs()
 			if err != nil {
 				return fmt.Errorf("can't build profiles configuration: %w", err)
 			}
@@ -183,25 +196,21 @@ func NewNotificationsInspectCmd() *cobra.Command {
 		Example:           `cscli notifications inspect <plugin_name>`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, arg []string) error {
-			var (
-				cfg NotificationsCfg
-				ok  bool
-			)
-
-			pluginName := arg[0]
-
-			if pluginName == "" {
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if args[0] == "" {
 				return fmt.Errorf("please provide a plugin name to inspect")
 			}
-			ncfgs, err := getNotificationsConfiguration()
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ncfgs, err := getProfilesConfigs()
 			if err != nil {
 				return fmt.Errorf("can't build profiles configuration: %w", err)
 			}
-			if cfg, ok = ncfgs[pluginName]; !ok {
-				return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName)
+			cfg, ok := ncfgs[args[0]]
+			if !ok {
+				return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
 			}
-
 			if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
 				fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
 				fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
@@ -224,75 +233,125 @@ func NewNotificationsInspectCmd() *cobra.Command {
 	return cmdNotificationsInspect
 }
 
+func NewNotificationsTestCmd() *cobra.Command {
+	var (
+		pluginBroker  csplugin.PluginBroker
+		pluginTomb    tomb.Tomb
+		alertOverride string
+	)
+	var cmdNotificationsTest = &cobra.Command{
+		Use:               "test [plugin name]",
+		Short:             "send a generic test alert to notification plugin",
+		Long:              `send a generic test alert to a notification plugin to test configuration even if is not active`,
+		Example:           `cscli notifications test [plugin_name]`,
+		Args:              cobra.ExactArgs(1),
+		DisableAutoGenTag: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			pconfigs, err := getPluginConfigs()
+			if err != nil {
+				return fmt.Errorf("can't build profiles configuration: %w", err)
+			}
+			cfg, ok := pconfigs[args[0]]
+			if !ok {
+				return fmt.Errorf("plugin name: '%s' does not exist", args[0])
+			}
+			//Create a single profile with plugin name as notification name
+			return pluginBroker.Init(csConfig.PluginConfig, []*csconfig.ProfileCfg{
+				{
+					Notifications: []string{
+						cfg.Name,
+					},
+				},
+			}, csConfig.ConfigPaths)
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			pluginTomb.Go(func() error {
+				pluginBroker.Run(&pluginTomb)
+				return nil
+			})
+			alert := &models.Alert{
+				Capacity: ptr.Of(int32(0)),
+				Decisions: []*models.Decision{{
+					Duration: ptr.Of("4h"),
+					Scope:    ptr.Of("Ip"),
+					Value:    ptr.Of("10.10.10.10"),
+					Type:     ptr.Of("ban"),
+					Scenario: ptr.Of("test alert"),
+					Origin:   ptr.Of(types.CscliOrigin),
+				}},
+				Events:          []*models.Event{},
+				EventsCount:     ptr.Of(int32(1)),
+				Leakspeed:       ptr.Of("0"),
+				Message:         ptr.Of("test alert"),
+				ScenarioHash:    ptr.Of(""),
+				Scenario:        ptr.Of("test alert"),
+				ScenarioVersion: ptr.Of(""),
+				Simulated:       ptr.Of(false),
+				Source: &models.Source{
+					AsName:   "",
+					AsNumber: "",
+					Cn:       "",
+					IP:       "10.10.10.10",
+					Range:    "",
+					Scope:    ptr.Of("Ip"),
+					Value:    ptr.Of("10.10.10.10"),
+				},
+				StartAt:   ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+				StopAt:    ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+				CreatedAt: time.Now().UTC().Format(time.RFC3339),
+			}
+			if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
+				return fmt.Errorf("failed to unmarshal alert override: %w", err)
+			}
+			pluginBroker.PluginChannel <- csplugin.ProfileAlert{
+				ProfileID: uint(0),
+				Alert:     alert,
+			}
+			//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
+			pluginTomb.Kill(fmt.Errorf("terminating"))
+			pluginTomb.Wait()
+			return nil
+		},
+	}
+	cmdNotificationsTest.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
+
+	return cmdNotificationsTest
+}
+
 func NewNotificationsReinjectCmd() *cobra.Command {
-	var remediation bool
 	var alertOverride string
+	var alert *models.Alert
 
 	var cmdNotificationsReinject = &cobra.Command{
 		Use:   "reinject",
-		Short: "reinject alert into notifications system",
-		Long:  `Reinject alert into notifications system`,
+		Short: "reinject an alert into profiles to trigger notifications",
+		Long:  `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
 		Example: `
 cscli notifications reinject <alert_id>
-cscli notifications reinject <alert_id> --remediation
+cscli notifications reinject <alert_id> -a '{"remediation": false,"scenario":"notification/test"}'
 cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"notification/test"}'
 `,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			alert, err = FetchAlertFromArgString(args[0])
+			if err != nil {
+				return err
+			}
+			return nil
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var (
 				pluginBroker csplugin.PluginBroker
 				pluginTomb   tomb.Tomb
 			)
-			if len(args) != 1 {
-				printHelp(cmd)
-				return fmt.Errorf("wrong number of argument: there should be one argument")
-			}
-
-			//first: get the alert
-			id, err := strconv.Atoi(args[0])
-			if err != nil {
-				return fmt.Errorf("bad alert id %s", args[0])
-			}
-			if err := csConfig.LoadAPIClient(); err != nil {
-				return fmt.Errorf("loading api client: %w", err)
-			}
-			if csConfig.API.Client == nil {
-				return fmt.Errorf("missing configuration on 'api_client:'")
-			}
-			if csConfig.API.Client.Credentials == nil {
-				return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath)
-			}
-			apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
-			if err != nil {
-				return fmt.Errorf("error parsing the URL of the API: %w", err)
-			}
-			client, err := apiclient.NewClient(&apiclient.Config{
-				MachineID:     csConfig.API.Client.Credentials.Login,
-				Password:      strfmt.Password(csConfig.API.Client.Credentials.Password),
-				UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
-				URL:           apiURL,
-				VersionPrefix: "v1",
-			})
-			if err != nil {
-				return fmt.Errorf("error creating the client for the API: %w", err)
-			}
-			alert, _, err := client.Alerts.GetByID(context.Background(), id)
-			if err != nil {
-				return fmt.Errorf("can't find alert with id %s: %w", args[0], err)
-			}
-
 			if alertOverride != "" {
-				if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
+				if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
 					return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
 				}
 			}
-			if !remediation {
-				alert.Remediation = true
-			}
-
-			// second we start plugins
-			err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
+			err := pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
 			if err != nil {
 				return fmt.Errorf("can't initialize plugins: %w", err)
 			}
@@ -302,8 +361,6 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 				return nil
 			})
 
-			//third: get the profile(s), and process the whole stuff
-
 			profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
 			if err != nil {
 				return fmt.Errorf("cannot extract profiles from configuration: %w", err)
@@ -338,15 +395,39 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 					break
 				}
 			}
-
-			//			time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
+			//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
 			pluginTomb.Kill(fmt.Errorf("terminating"))
 			pluginTomb.Wait()
 			return nil
 		},
 	}
-	cmdNotificationsReinject.Flags().BoolVarP(&remediation, "remediation", "r", false, "Set Alert.Remediation to false in the reinjected alert (see your profile filter configuration)")
 	cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
 
 	return cmdNotificationsReinject
 }
+
+func FetchAlertFromArgString(toParse string) (*models.Alert, error) {
+	id, err := strconv.Atoi(toParse)
+	if err != nil {
+		return nil, fmt.Errorf("bad alert id %s", toParse)
+	}
+	apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
+	}
+	client, err := apiclient.NewClient(&apiclient.Config{
+		MachineID:     csConfig.API.Client.Credentials.Login,
+		Password:      strfmt.Password(csConfig.API.Client.Credentials.Password),
+		UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
+		URL:           apiURL,
+		VersionPrefix: "v1",
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error creating the client for the API: %w", err)
+	}
+	alert, _, err := client.Alerts.GetByID(context.Background(), id)
+	if err != nil {
+		return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
+	}
+	return alert, nil
+}

cmd/crowdsec-cli/parsers.go

@@ -1,205 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewParsersCmd() *cobra.Command {
-	var cmdParsers = &cobra.Command{
-		Use:   "parsers [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
-		Example: `cscli parsers install crowdsecurity/sshd-logs
-cscli parsers inspect crowdsecurity/sshd-logs
-cscli parsers upgrade crowdsecurity/sshd-logs
-cscli parsers list
-cscli parsers remove crowdsecurity/sshd-logs
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"parser"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				return err
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("while setting hub branch: %w", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				return fmt.Errorf("failed to get hub index: %w", err)
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdParsers.AddCommand(NewParsersInstallCmd())
-	cmdParsers.AddCommand(NewParsersRemoveCmd())
-	cmdParsers.AddCommand(NewParsersUpgradeCmd())
-	cmdParsers.AddCommand(NewParsersInspectCmd())
-	cmdParsers.AddCommand(NewParsersListCmd())
-
-	return cmdParsers
-}
-
-func NewParsersInstallCmd() *cobra.Command {
-	var ignoreError bool
-
-	var cmdParsersInstall = &cobra.Command{
-		Use:               "install [config]",
-		Short:             "Install given parser(s)",
-		Long:              `Fetch and install given parser(s) from hub`,
-		Example:           `cscli parsers install crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS, name)
-					Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
-
-	return cmdParsersInstall
-}
-
-func NewParsersRemoveCmd() *cobra.Command {
-	cmdParsersRemove := &cobra.Command{
-		Use:               "remove [config]",
-		Short:             "Remove given parser(s)",
-		Long:              `Remove given parse(s) from hub`,
-		Example:           `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one parser to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
-			}
-
-			return nil
-		},
-	}
-
-	cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
-
-	return cmdParsersRemove
-}
-
-func NewParsersUpgradeCmd() *cobra.Command {
-	cmdParsersUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given parser(s)",
-		Long:              `Fetch and upgrade given parser(s) from hub`,
-		Example:           `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one parser to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdParsersUpgrade
-}
-
-func NewParsersInspectCmd() *cobra.Command {
-	var cmdParsersInspect = &cobra.Command{
-		Use:               "inspect [name]",
-		Short:             "Inspect given parser",
-		Long:              `Inspect given parser`,
-		Example:           `cscli parsers inspect crowdsec/xxx`,
-		DisableAutoGenTag: true,
-		Args:              cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS)
-		},
-	}
-
-	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-
-	return cmdParsersInspect
-}
-
-func NewParsersListCmd() *cobra.Command {
-	var cmdParsersList = &cobra.Command{
-		Use:   "list [name]",
-		Short: "List all parsers or given one",
-		Long:  `List all parsers or given one`,
-		Example: `cscli parsers list
-cscli parser list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
-		},
-	}
-
-	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdParsersList
-}

cmd/crowdsec-cli/postoverflows.go

@@ -1,202 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewPostOverflowsCmd() *cobra.Command {
-	cmdPostOverflows := &cobra.Command{
-		Use:   "postoverflows [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
-		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
-		cscli postoverflows inspect crowdsecurity/cdn-whitelist
-		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
-		cscli postoverflows list
-		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"postoverflow"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				return err
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("while setting hub branch: %w", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				return fmt.Errorf("failed to get hub index: %w", err)
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
-
-	return cmdPostOverflows
-}
-
-func NewPostOverflowsInstallCmd() *cobra.Command {
-	var ignoreError bool
-
-	cmdPostOverflowsInstall := &cobra.Command{
-		Use:               "install [config]",
-		Short:             "Install given postoverflow(s)",
-		Long:              `Fetch and install given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
-					Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
-
-	return cmdPostOverflowsInstall
-}
-
-func NewPostOverflowsRemoveCmd() *cobra.Command {
-	cmdPostOverflowsRemove := &cobra.Command{
-		Use:               "remove [config]",
-		Short:             "Remove given postoverflow(s)",
-		Long:              `remove given postoverflow(s)`,
-		Example:           `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
-			}
-
-			return nil
-		},
-	}
-
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
-
-	return cmdPostOverflowsRemove
-}
-
-func NewPostOverflowsUpgradeCmd() *cobra.Command {
-	cmdPostOverflowsUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given postoverflow(s)",
-		Long:              `Fetch and Upgrade given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdPostOverflowsUpgrade
-}
-
-func NewPostOverflowsInspectCmd() *cobra.Command {
-	cmdPostOverflowsInspect := &cobra.Command{
-		Use:               "inspect [config]",
-		Short:             "Inspect given postoverflow",
-		Long:              `Inspect given postoverflow`,
-		Example:           `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		Args:              cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS_OVFLW)
-		},
-	}
-
-	return cmdPostOverflowsInspect
-}
-
-func NewPostOverflowsListCmd() *cobra.Command {
-	cmdPostOverflowsList := &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all postoverflows or given one",
-		Long:  `List all postoverflows or given one`,
-		Example: `cscli postoverflows list
-cscli postoverflows list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
-		},
-	}
-
-	cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdPostOverflowsList
-}

cmd/crowdsec-cli/require/branch.go

@@ -0,0 +1,58 @@
+package require
+
+// Set the appropriate hub branch according to config settings and crowdsec version
+
+import (
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+)
+
+func chooseBranch(cfg *csconfig.Config) string {
+	// this was set from config.yaml or flag
+	if cfg.Cscli.HubBranch != "" {
+		log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
+		return cfg.Cscli.HubBranch
+	}
+
+	latest, err := cwversion.Latest()
+	if err != nil {
+		log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
+		return "master"
+	}
+
+	csVersion := cwversion.VersionStrip()
+	if csVersion == latest {
+		log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
+		return "master"
+	}
+
+	// if current version is greater than the latest we are in pre-release
+	if semver.Compare(csVersion, latest) == 1 {
+		log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
+		return "master"
+	}
+
+	if csVersion == "" {
+		log.Warning("Crowdsec version is not set, using hub branch 'master'")
+		return "master"
+	}
+
+	log.Warnf("A new CrowdSec release is available (%s). "+
+		"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
+		latest, csVersion)
+	return csVersion
+}
+
+
+// HubBranch sets the branch (in cscli config) and returns its value
+// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
+func HubBranch(cfg *csconfig.Config) string {
+	branch := chooseBranch(cfg)
+
+	cfg.Cscli.HubBranch = branch
+
+	return branch
+}

cmd/crowdsec-cli/require/require.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
 func LAPI(c *csconfig.Config) error {
@@ -22,6 +23,7 @@ func CAPI(c *csconfig.Config) error {
 	if c.API.Server.OnlineClient == nil {
 		return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
 	}
+
 	return nil
 }
 
@@ -29,6 +31,7 @@ func PAPI(c *csconfig.Config) error {
 	if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
 		return fmt.Errorf("no PAPI URL in configuration")
 	}
+
 	return nil
 }
 
@@ -44,6 +47,7 @@ func DB(c *csconfig.Config) error {
 	if err := c.LoadDBConfig(); err != nil {
 		return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
 	}
+
 	return nil
 }
 
@@ -63,3 +67,33 @@ func Notifications(c *csconfig.Config) error {
 	return nil
 }
 
+// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
+func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
+	// set branch in config, and log if necessary
+	branch := HubBranch(c)
+	remote := &cwhub.RemoteHubCfg {
+		Branch: branch,
+		URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
+		// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
+		IndexPath: ".index.json",
+	}
+
+	return remote
+}
+
+// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
+// If no remote parameter is provided, the hub can only be used for local operations.
+func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
+	local := c.Hub
+
+	if local == nil {
+		return nil, fmt.Errorf("you must configure cli before interacting with hub")
+	}
+
+	hub, err := cwhub.NewHub(local, remote, false)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
+	}
+
+	return hub, nil
+}

cmd/crowdsec-cli/scenarios.go

@@ -1,199 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewScenariosCmd() *cobra.Command {
-	var cmdScenarios = &cobra.Command{
-		Use:   "scenarios [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
-		Example: `cscli scenarios list [-a]
-cscli scenarios install crowdsecurity/ssh-bf
-cscli scenarios inspect crowdsecurity/ssh-bf
-cscli scenarios upgrade crowdsecurity/ssh-bf
-cscli scenarios remove crowdsecurity/ssh-bf
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"scenario"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				return err
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("while setting hub branch: %w", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				return fmt.Errorf("failed to get hub index: %w", err)
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdScenarios.AddCommand(NewCmdScenariosInstall())
-	cmdScenarios.AddCommand(NewCmdScenariosRemove())
-	cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
-	cmdScenarios.AddCommand(NewCmdScenariosInspect())
-	cmdScenarios.AddCommand(NewCmdScenariosList())
-
-	return cmdScenarios
-}
-
-func NewCmdScenariosInstall() *cobra.Command {
-	var ignoreError bool
-
-	var cmdScenariosInstall = &cobra.Command{
-		Use:     "install [config]",
-		Short:   "Install given scenario(s)",
-		Long:    `Fetch and install given scenario(s) from hub`,
-		Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.SCENARIOS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
-					Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
-
-	return cmdScenariosInstall
-}
-
-func NewCmdScenariosRemove() *cobra.Command {
-	var cmdScenariosRemove = &cobra.Command{
-		Use:     "remove [config]",
-		Short:   "Remove given scenario(s)",
-		Long:    `remove given scenario(s)`,
-		Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
-		Aliases: []string{"delete"},
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one scenario to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
-			}
-			return nil
-		},
-	}
-	cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
-
-	return cmdScenariosRemove
-}
-
-func NewCmdScenariosUpgrade() *cobra.Command {
-	var cmdScenariosUpgrade = &cobra.Command{
-		Use:     "upgrade [config]",
-		Short:   "Upgrade given scenario(s)",
-		Long:    `Fetch and Upgrade given scenario(s) from hub`,
-		Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
-	cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdScenariosUpgrade
-}
-
-func NewCmdScenariosInspect() *cobra.Command {
-	var cmdScenariosInspect = &cobra.Command{
-		Use:     "inspect [config]",
-		Short:   "Inspect given scenario",
-		Long:    `Inspect given scenario`,
-		Example: `cscli scenarios inspect crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.SCENARIOS)
-		},
-	}
-	cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-
-	return cmdScenariosInspect
-}
-
-func NewCmdScenariosList() *cobra.Command {
-	var cmdScenariosList = &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all scenario(s) or given one",
-		Long:  `List all scenario(s) or given one`,
-		Example: `cscli scenarios list
-cscli scenarios list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
-		},
-	}
-	cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdScenariosList
-}

cmd/crowdsec-cli/setup.go

@@ -6,13 +6,15 @@ import (
 	"os"
 	"os/exec"
 
+	goccyyaml "github.com/goccy/go-yaml"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
-	goccyyaml "github.com/goccy/go-yaml"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/setup"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 // NewSetupCmd defines the "cscli setup" command.
@@ -303,7 +305,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("while reading file %s: %w", fromFile, err)
 	}
 
-	if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
+		return err
+	}
+
+	if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/simulation.go

@@ -6,9 +6,10 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
+	"slices"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -18,7 +19,7 @@ func addToExclusion(name string) error {
 }
 
 func removeFromExclusion(name string) error {
-	index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
+	index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
 
 	// Remove element from the slice
 	csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
@@ -111,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
 			if err := csConfig.LoadSimulation(); err != nil {
 				log.Fatal(err)
 			}
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before using simulation")
-			}
 			if csConfig.Cscli.SimulationConfig == nil {
 				return fmt.Errorf("no simulation configured")
 			}
@@ -144,22 +142,19 @@ func NewSimulationEnableCmd() *cobra.Command {
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := csConfig.LoadHub(); err != nil {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
 				log.Fatal(err)
 			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
 
 			if len(args) > 0 {
 				for _, scenario := range args {
-					var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
+					var item = hub.GetItem(cwhub.SCENARIOS, scenario)
 					if item == nil {
 						log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
 						continue
 					}
-					if !item.Installed {
+					if !item.State.Installed {
 						log.Warningf("'%s' isn't enabled", scenario)
 					}
 					isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)

cmd/crowdsec-cli/support.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@@ -57,10 +58,6 @@ func stripAnsiString(str string) string {
 
 func collectMetrics() ([]byte, []byte, error) {
 	log.Info("Collecting prometheus metrics")
-	err := csConfig.LoadPrometheus()
-	if err != nil {
-		return nil, nil, err
-	}
 
 	if csConfig.Cscli.PrometheusUrl == "" {
 		log.Warn("No Prometheus URL configured, metrics will not be collected")
@@ -68,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) {
 	}
 
 	humanMetrics := bytes.NewBuffer(nil)
-	err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
+	err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
 
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
 	}
 
-	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
+	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
 	}
@@ -131,28 +128,21 @@ func collectOSInfo() ([]byte, error) {
 	return w.Bytes(), nil
 }
 
-func initHub() error {
-	if err := csConfig.LoadHub(); err != nil {
-		return fmt.Errorf("cannot load hub: %s", err)
-	}
-	if csConfig.Hub == nil {
-		return fmt.Errorf("hub not configured")
-	}
+func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
+	var err error
 
-	if err := cwhub.SetHubBranch(); err != nil {
-		return fmt.Errorf("cannot set hub branch: %s", err)
-	}
+	out := bytes.NewBuffer(nil)
+	log.Infof("Collecting %s list", itemType)
+
+	items := make(map[string][]*cwhub.Item)
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("no hub index found: %s", err)
+	if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
 	}
-	return nil
-}
 
-func collectHubItems(itemType string) []byte {
-	out := bytes.NewBuffer(nil)
-	log.Infof("Collecting %s list", itemType)
-	ListItems(out, []string{itemType}, []string{}, false, true, all)
+	if err := listItems(out, []string{itemType}, items); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
 	return out.Bytes()
 }
 
@@ -174,7 +164,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
 	return out.Bytes(), nil
 }
 
-func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
+func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
 	if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
 		return []byte("No agent credentials found, are we LAPI ?")
 	}
@@ -184,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 	if err != nil {
 		return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
 	}
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
 	}
@@ -312,7 +302,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				skipAgent = true
 			}
 
-			err = initHub()
+			hub, err := require.Hub(csConfig, nil)
 			if err != nil {
 				log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 				skipHub = true
@@ -351,10 +341,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
 			infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
 
 			if !skipHub {
-				infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
-				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
-				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
-				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
+				infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
+				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
+				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
+				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
 			}
 
 			if !skipDB {
@@ -376,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
 					csConfig.API.Server.OnlineClient.Credentials.Password,
 					csConfig.API.Server.OnlineClient.Credentials.URL,
-					CAPIURLPrefix)
+					CAPIURLPrefix,
+					hub)
 			}
 
 			if !skipLAPI {
@@ -384,7 +375,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
 					csConfig.API.Client.Credentials.Password,
 					csConfig.API.Client.Credentials.URL,
-					LAPIURLPrefix)
+					LAPIURLPrefix,
+					hub)
 				infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
 			}
 

cmd/crowdsec-cli/utils.go

@@ -1,36 +1,17 @@
 package main
 
 import (
-	"encoding/csv"
-	"encoding/json"
 	"fmt"
-	"io"
-	"math"
 	"net"
-	"net/http"
-	"os"
-	"strconv"
 	"strings"
-	"time"
 
-	"github.com/fatih/color"
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/agext/levenshtein"
-	"golang.org/x/exp/slices"
-	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-const MaxDistance = 7
-
 func printHelp(cmd *cobra.Command) {
 	err := cmd.Help()
 	if err != nil {
@@ -38,238 +19,6 @@ func printHelp(cmd *cobra.Command) {
 	}
 }
 
-func indexOf(s string, slice []string) int {
-	for i, elem := range slice {
-		if s == elem {
-			return i
-		}
-	}
-	return -1
-}
-
-func LoadHub() error {
-	if err := csConfig.LoadHub(); err != nil {
-		log.Fatal(err)
-	}
-	if csConfig.Hub == nil {
-		return fmt.Errorf("unable to load hub")
-	}
-
-	if err := cwhub.SetHubBranch(); err != nil {
-		log.Warningf("unable to set hub branch (%s), default to master", err)
-	}
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
-	}
-
-	return nil
-}
-
-func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
-	errMsg := ""
-	if score < MaxDistance {
-		errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
-	} else {
-		errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
-	}
-	if ignoreErr {
-		log.Error(errMsg)
-	} else {
-		log.Fatalf(errMsg)
-	}
-}
-
-func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
-	allItems := make([]string, 0)
-	nearestScore := 100
-	nearestItem := &cwhub.Item{}
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		allItems = append(allItems, item.Name)
-	}
-
-	for _, s := range allItems {
-		d := levenshtein.Distance(itemName, s, nil)
-		if d < nearestScore {
-			nearestScore = d
-			nearestItem = cwhub.GetItem(itemType, s)
-		}
-	}
-	return nearestItem, nearestScore
-}
-
-func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := LoadHub(); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
-			comp = append(comp, item.Name)
-		}
-	}
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := LoadHub(); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	var items []string
-	var err error
-	switch itemType {
-	case cwhub.PARSERS:
-		items, err = cwhub.GetInstalledParsersAsString()
-	case cwhub.SCENARIOS:
-		items, err = cwhub.GetInstalledScenariosAsString()
-	case cwhub.PARSERS_OVFLW:
-		items, err = cwhub.GetInstalledPostOverflowsAsString()
-	case cwhub.COLLECTIONS:
-		items, err = cwhub.GetInstalledCollectionsAsString()
-	default:
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	if err != nil {
-		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-	comp := make([]string, 0)
-
-	if toComplete != "" {
-		for _, item := range items {
-			if strings.Contains(item, toComplete) {
-				comp = append(comp, item)
-			}
-		}
-	} else {
-		comp = items
-	}
-
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
-	var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
-
-	for _, itemType := range itemTypes {
-		itemName := ""
-		if len(args) == 1 {
-			itemName = args[0]
-		}
-		hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
-	}
-
-	if csConfig.Cscli.Output == "human" {
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
-		}
-	} else if csConfig.Cscli.Output == "json" {
-		x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
-		if err != nil {
-			log.Fatalf("failed to unmarshal")
-		}
-		out.Write(x)
-	} else if csConfig.Cscli.Output == "raw" {
-		csvwriter := csv.NewWriter(out)
-		if showHeader {
-			header := []string{"name", "status", "version", "description"}
-			if showType {
-				header = append(header, "type")
-			}
-			err := csvwriter.Write(header)
-			if err != nil {
-				log.Fatalf("failed to write header: %s", err)
-			}
-
-		}
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			for _, status := range statuses {
-				if status.LocalVersion == "" {
-					status.LocalVersion = "n/a"
-				}
-				row := []string{
-					status.Name,
-					status.Status,
-					status.LocalVersion,
-					status.Description,
-				}
-				if showType {
-					row = append(row, itemType)
-				}
-				err := csvwriter.Write(row)
-				if err != nil {
-					log.Fatalf("failed to write raw output : %s", err)
-				}
-			}
-		}
-		csvwriter.Flush()
-	}
-}
-
-func InspectItem(name string, objecitemType string) {
-
-	hubItem := cwhub.GetItem(objecitemType, name)
-	if hubItem == nil {
-		log.Fatalf("unable to retrieve item.")
-	}
-	var b []byte
-	var err error
-	switch csConfig.Cscli.Output {
-	case "human", "raw":
-		b, err = yaml.Marshal(*hubItem)
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	case "json":
-		b, err = json.MarshalIndent(*hubItem, "", " ")
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	}
-	fmt.Printf("%s", string(b))
-	if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
-		return
-	}
-
-	if prometheusURL == "" {
-		//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
-		//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
-		listenAddr := csConfig.Prometheus.ListenAddr
-		if listenAddr == "" {
-			listenAddr = "127.0.0.1"
-		}
-		listenPort := csConfig.Prometheus.ListenPort
-		if listenPort == 0 {
-			listenPort = 6060
-		}
-		prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
-		log.Debugf("No prometheus URL provided using: %s", prometheusURL)
-	}
-
-	fmt.Printf("\nCurrent metrics : \n")
-	ShowMetrics(hubItem)
-}
-
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
 
 	/*if a range is provided, change the scope*/
@@ -300,417 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 	return nil
 }
 
-func ShowMetrics(hubItem *cwhub.Item) {
-	switch hubItem.Type {
-	case cwhub.PARSERS:
-		metrics := GetParserMetric(prometheusURL, hubItem.Name)
-		parserMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.SCENARIOS:
-		metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
-		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.COLLECTIONS:
-		for _, item := range hubItem.Parsers {
-			metrics := GetParserMetric(prometheusURL, item)
-			parserMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Scenarios {
-			metrics := GetScenarioMetric(prometheusURL, item)
-			scenarioMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Collections {
-			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
-			if hubItem == nil {
-				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
-			}
-			ShowMetrics(hubItem)
-		}
-	default:
-		log.Errorf("item of type '%s' is unknown", hubItem.Type)
-	}
-}
-
-// GetParserMetric is a complete rip from prom2json
-func GetParserMetric(url string, itemName string) map[string]map[string]int {
-	stats := make(map[string]map[string]int)
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			source, ok := metric.Labels["source"]
-			if !ok {
-				log.Debugf("no source in Metric %v", metric.Labels)
-			} else {
-				if srctype, ok := metric.Labels["type"]; ok {
-					source = srctype + ":" + source
-				}
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_reader_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-					stats[source]["parsed"] = 0
-					stats[source]["reads"] = 0
-					stats[source]["unparsed"] = 0
-					stats[source]["hits"] = 0
-				}
-				stats[source]["reads"] += ival
-			case "cs_parser_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_parser_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			case "cs_node_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["hits"] += ival
-			case "cs_node_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_node_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetScenarioMetric(url string, itemName string) map[string]int {
-	stats := make(map[string]int)
-
-	stats["instantiation"] = 0
-	stats["curr_count"] = 0
-	stats["overflow"] = 0
-	stats["pour"] = 0
-	stats["underflow"] = 0
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_bucket_created_total":
-				stats["instantiation"] += ival
-			case "cs_buckets":
-				stats["curr_count"] += ival
-			case "cs_bucket_overflowed_total":
-				stats["overflow"] += ival
-			case "cs_bucket_poured_total":
-				stats["pour"] += ival
-			case "cs_bucket_underflowed_total":
-				stats["underflow"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-// it's a rip of the cli version, but in silent-mode
-func silenceInstallItem(name string, obtype string) (string, error) {
-	var item = cwhub.GetItem(obtype, name)
-	if item == nil {
-		return "", fmt.Errorf("error retrieving item")
-	}
-	it := *item
-	if downloadOnly && it.Downloaded && it.UpToDate {
-		return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
-	}
-	it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
-	if err != nil {
-		return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, it); err != nil {
-		return "", err
-	}
-
-	if downloadOnly {
-		return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
-	}
-	it, err = cwhub.EnableItem(csConfig.Hub, it)
-	if err != nil {
-		return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, it); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("Enabled %s", it.Name), nil
-}
-
-func GetPrometheusMetric(url string) []*prom2json.Family {
-	mfChan := make(chan *dto.MetricFamily, 1024)
-
-	// Start with the DefaultTransport for sane defaults.
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	// Conservatively disable HTTP keep-alives as this program will only
-	// ever need a single HTTP request.
-	transport.DisableKeepAlives = true
-	// Timeout early if the server doesn't even return the headers.
-	transport.ResponseHeaderTimeout = time.Minute
-
-	go func() {
-		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
-		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
-		if err != nil {
-			log.Fatalf("failed to fetch prometheus metrics : %v", err)
-		}
-	}()
-
-	result := []*prom2json.Family{}
-	for mf := range mfChan {
-		result = append(result, prom2json.NewFamily(mf))
-	}
-	log.Debugf("Finished reading prometheus output, %d entries", len(result))
-
-	return result
-}
-
-func RestoreHub(dirPath string) error {
-	var err error
-
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-	if err := cwhub.SetHubBranch(); err != nil {
-		return fmt.Errorf("error while setting hub branch: %s", err)
-	}
-
-	for _, itype := range cwhub.ItemTypes {
-		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
-		if _, err = os.Stat(itemDirectory); err != nil {
-			log.Infof("no %s in backup", itype)
-			continue
-		}
-		/*restore the upstream items*/
-		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
-		file, err := os.ReadFile(upstreamListFN)
-		if err != nil {
-			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
-		}
-		var upstreamList []string
-		err = json.Unmarshal(file, &upstreamList)
-		if err != nil {
-			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
-		}
-		for _, toinstall := range upstreamList {
-			label, err := silenceInstallItem(toinstall, itype)
-			if err != nil {
-				log.Errorf("Error while installing %s : %s", toinstall, err)
-			} else if label != "" {
-				log.Infof("Installed %s : %s", toinstall, label)
-			} else {
-				log.Printf("Installed %s : ok", toinstall)
-			}
-		}
-
-		/*restore the local and tainted items*/
-		files, err := os.ReadDir(itemDirectory)
-		if err != nil {
-			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
-		}
-		for _, file := range files {
-			//this was the upstream data
-			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
-				continue
-			}
-			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
-				//we expect a stage here
-				if !file.IsDir() {
-					continue
-				}
-				stage := file.Name()
-				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
-				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
-				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
-					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
-				}
-				/*find items*/
-				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
-				if err != nil {
-					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
-				}
-				//finally copy item
-				for _, tfile := range ifiles {
-					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
-					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
-					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
-					if err = CopyFile(sourceFile, destinationFile); err != nil {
-						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
-					}
-					log.Infof("restored %s to %s", sourceFile, destinationFile)
-				}
-			} else {
-				log.Infof("Going to restore local/tainted [%s]", file.Name())
-				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
-				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
-				if err = CopyFile(sourceFile, destinationFile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
-				}
-				log.Infof("restored %s to %s", sourceFile, destinationFile)
-			}
-
-		}
-	}
-	return nil
-}
-
-func BackupHub(dirPath string) error {
-	var err error
-	var itemDirectory string
-	var upstreamParsers []string
-
-	for _, itemType := range cwhub.ItemTypes {
-		clog := log.WithFields(log.Fields{
-			"type": itemType,
-		})
-		itemMap := cwhub.GetItemMap(itemType)
-		if itemMap == nil {
-			clog.Infof("No %s to backup.", itemType)
-			continue
-		}
-		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
-		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
-			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
-		}
-		upstreamParsers = []string{}
-		for k, v := range itemMap {
-			clog = clog.WithFields(log.Fields{
-				"file": v.Name,
-			})
-			if !v.Installed { //only backup installed ones
-				clog.Debugf("[%s] : not installed", k)
-				continue
-			}
-
-			//for the local/tainted ones, we backup the full file
-			if v.Tainted || v.Local || !v.UpToDate {
-				//we need to backup stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
-					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
-					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
-						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
-					}
-				}
-				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
-				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
-				if err = CopyFile(v.LocalPath, tfile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
-				}
-				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
-				continue
-			}
-			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
-			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
-			upstreamParsers = append(upstreamParsers, v.Name)
-		}
-		//write the upstream items
-		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
-		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
-		if err != nil {
-			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
-		}
-		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
-		if err != nil {
-			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
-		}
-		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
-	}
-
-	return nil
-}
-
-type unit struct {
-	value  int64
-	symbol string
-}
-
-var ranges = []unit{
-	{value: 1e18, symbol: "E"},
-	{value: 1e15, symbol: "P"},
-	{value: 1e12, symbol: "T"},
-	{value: 1e9,  symbol: "G"},
-	{value: 1e6,  symbol: "M"},
-	{value: 1e3,  symbol: "k"},
-	{value: 1,    symbol: ""},
-}
-
-func formatNumber(num int) string {
-	goodUnit := unit{}
-	for _, u := range ranges {
-		if int64(num) >= u.value {
-			goodUnit = u
-			break
-		}
-	}
-
-	if goodUnit.value == 1 {
-		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
-	}
-
-	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
-	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
-}
-
 func getDBClient() (*database.Client, error) {
 	var err error
 	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
@@ -744,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
 	}
 
 	return slice
-
 }

cmd/crowdsec-cli/utils_table.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"strconv"
 
 	"github.com/aquasecurity/table"
 	"github.com/enescakir/emoji"
@@ -10,14 +11,15 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
+func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
 	t := newLightTable(out)
 	t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	for _, status := range statuses {
-		t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
+	for _, item := range items {
+		status, emo := item.InstallStatus()
+		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.State.LocalVersion, item.State.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()
@@ -31,11 +33,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
 	t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
 
 	t.AddRow(
-		fmt.Sprintf("%d", metrics["curr_count"]),
-		fmt.Sprintf("%d", metrics["overflow"]),
-		fmt.Sprintf("%d", metrics["instantiation"]),
-		fmt.Sprintf("%d", metrics["pour"]),
-		fmt.Sprintf("%d", metrics["underflow"]),
+		strconv.Itoa(metrics["curr_count"]),
+		strconv.Itoa(metrics["overflow"]),
+		strconv.Itoa(metrics["instantiation"]),
+		strconv.Itoa(metrics["pour"]),
+		strconv.Itoa(metrics["underflow"]),
 	)
 
 	renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@@ -43,23 +45,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
 }
 
 func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
-	skip := true
 	t := newTable(out)
 	t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
 
+	// don't show table if no hits
+	showTable := false
+
 	for source, stats := range metrics {
 		if stats["hits"] > 0 {
 			t.AddRow(
 				source,
-				fmt.Sprintf("%d", stats["hits"]),
-				fmt.Sprintf("%d", stats["parsed"]),
-				fmt.Sprintf("%d", stats["unparsed"]),
+				strconv.Itoa(stats["hits"]),
+				strconv.Itoa(stats["parsed"]),
+				strconv.Itoa(stats["unparsed"]),
 			)
-			skip = false
+			showTable = true
 		}
 	}
 
-	if !skip {
+	if showTable {
 		renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
 		t.Render()
 	}

cmd/crowdsec/crowdsec.go

@@ -20,21 +20,16 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
+func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
 	var err error
 
-	// Populate cwhub package tools
-	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
-		return nil, fmt.Errorf("while loading hub index: %w", err)
-	}
-
 	// Start loading configs
-	csParsers := parser.NewParsers()
+	csParsers := parser.NewParsers(hub)
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
 		return nil, fmt.Errorf("while loading parsers: %w", err)
 	}
 
-	if err := LoadBuckets(cConfig); err != nil {
+	if err := LoadBuckets(cConfig, hub); err != nil {
 		return nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 
@@ -44,7 +39,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	return csParsers, nil
 }
 
-func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
+func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
 	inputEventChan = make(chan types.Event)
 	inputLineChan = make(chan types.Event)
 
@@ -99,7 +94,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 		for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
 			outputsTomb.Go(func() error {
 				defer trace.CatchPanic("crowdsec/runOutput")
-				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
+				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
 					log.Fatalf("starting outputs error : %s", err)
 					return err
 				}
@@ -131,7 +126,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 	return nil
 }
 
-func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
+func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
 	crowdsecTomb.Go(func() error {
 		defer trace.CatchPanic("crowdsec/serveCrowdsec")
 		go func() {
@@ -139,7 +134,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
 			// this logs every time, even at config reload
 			log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
 			agentReady <- true
-			if err := runCrowdsec(cConfig, parsers); err != nil {
+			if err := runCrowdsec(cConfig, parsers, hub); err != nil {
 				log.Fatalf("unable to start crowdsec routines: %s", err)
 			}
 		}()

cmd/crowdsec/main.go

@@ -75,20 +75,20 @@ type Flags struct {
 
 type labelsMap map[string]string
 
-func LoadBuckets(cConfig *csconfig.Config) error {
+func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
 	var (
 		err   error
 		files []string
 	)
-	for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
-		if hubScenarioItem.Installed {
-			files = append(files, hubScenarioItem.LocalPath)
+	for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
+		if hubScenarioItem.State.Installed {
+			files = append(files, hubScenarioItem.State.LocalPath)
 		}
 	}
 	buckets = leakybucket.NewBuckets()
 
 	log.Infof("Loading %d scenario files", len(files))
-	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
+	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
 
 	if err != nil {
 		return fmt.Errorf("scenario loading failed: %v", err)
@@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
 	cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
 	if err != nil {
-		return nil, err
-	}
-
-	if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
-		return nil, fmt.Errorf("unable to load configuration: common section is empty")
+		return nil, fmt.Errorf("while loading configuration file: %w", err)
 	}
 
 	cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		dumpStates = true
 	}
 
-	// Configuration paths are dependency to load crowdsec configuration
-	if err := cConfig.LoadConfigurationPaths(); err != nil {
-		return nil, err
-	}
-
 	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
 		// if we're in time-machine mode, we don't want to log to file
 		cConfig.Common.LogMedia = "stdout"

cmd/crowdsec/metrics.go

@@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 	if !config.Enabled {
 		return
 	}
-	if config.ListenAddr == "" {
-		log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
-		config.ListenAddr = "127.0.0.1"
-	}
-	if config.ListenPort == 0 {
-		log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
-		config.ListenPort = 6060
-	}
 
 	// Registering prometheus
 	// If in aggregated mode, do not register events associated with a source, to keep the cardinality low

cmd/crowdsec/output.go

@@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
 var bucketOverflows []types.Event
 
 func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
-	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
+	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
+	apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
 
 	var err error
 	ticker := time.NewTicker(1 * time.Second)
@@ -70,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	var cache []types.RuntimeAlert
 	var cacheMutex sync.Mutex
 
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		return fmt.Errorf("loading list of installed hub scenarios: %w", err)
 	}
@@ -93,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 		URL:            apiURL,
 		PapiURL:        papiURL,
 		VersionPrefix:  "v1",
-		UpdateScenario: cwhub.GetInstalledScenariosAsString,
+		UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)},
 	})
 	if err != nil {
 		return fmt.Errorf("new client api: %w", err)
@@ -145,13 +146,6 @@ LOOP:
 			}
 			break LOOP
 		case event := <-overflow:
-			//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
-			if dumpStates && event.Overflow.Alert != nil {
-				if bucketOverflows == nil {
-					bucketOverflows = make([]types.Event, 0)
-				}
-				bucketOverflows = append(bucketOverflows, event)
-			}
 			/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
 			if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
 				buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@@ -163,6 +157,14 @@ LOOP:
 				return fmt.Errorf("postoverflow failed : %s", err)
 			}
 			log.Printf("%s", *event.Overflow.Alert.Message)
+			//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
+			//dump after postoveflow processing to avoid missing whitelist info
+			if dumpStates && event.Overflow.Alert != nil {
+				if bucketOverflows == nil {
+					bucketOverflows = make([]types.Event, 0)
+				}
+				bucketOverflows = append(bucketOverflows, event)
+			}
 			if event.Overflow.Whitelisted {
 				log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
 				continue

cmd/crowdsec/serve.go

@@ -14,6 +14,7 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 	}
 
 	if !cConfig.DisableAgent {
-		csParsers, err := initCrowdsec(cConfig)
+		hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
+		if err != nil {
+			return nil, fmt.Errorf("while loading hub index: %w", err)
+		}
+
+		csParsers, err := initCrowdsec(cConfig, hub)
 		if err != nil {
 			return nil, fmt.Errorf("unable to init crowdsec: %w", err)
 		}
@@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 		}
 
 		agentReady := make(chan bool, 1)
-		serveCrowdsec(csParsers, cConfig, agentReady)
+		serveCrowdsec(csParsers, cConfig, hub, agentReady)
 	}
 
 	log.Printf("Reload is finished")
@@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
 	}
 
 	if !cConfig.DisableAgent {
-		csParsers, err := initCrowdsec(cConfig)
+		hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
+		if err != nil {
+			return fmt.Errorf("while loading hub index: %w", err)
+		}
+
+		csParsers, err := initCrowdsec(cConfig, hub)
 		if err != nil {
 			return fmt.Errorf("crowdsec init: %w", err)
 		}
 
 		// if it's just linting, we're done
 		if !flags.TestMode {
-			serveCrowdsec(csParsers, cConfig, agentReady)
+			serveCrowdsec(csParsers, cConfig, hub, agentReady)
 		}
 	} else {
 		agentReady <- true

config/config.yaml

@@ -6,7 +6,6 @@ common:
   log_max_size: 20
   compress_logs: true
   log_max_files: 10
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

config/config_win.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/config_win_no_lapi.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/dev.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: true
   log_media: stdout
   log_level: info
-  working_dir: .
 config_paths:
   config_dir: ./config
   data_dir: ./data/   

config/user.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data

docker/README.md

@@ -19,11 +19,7 @@ All the following images are available on Docker Hub for the architectures
 
  - `crowdsecurity/crowdsec:{version}`
 
-Recommended for production usage. Also available on GitHub (ghcr.io).
-
- - `crowdsecurity/crowdsec:dev`
-
-The latest stable release.
+Latest stable release recommended for production usage. Also available on GitHub (ghcr.io).
 
  - `crowdsecurity/crowdsec:dev`
 
@@ -190,6 +186,14 @@ It is not recommended anymore to bind-mount the full config.yaml file and you sh
 
 If you want to use the [notification system](https://docs.crowdsec.net/docs/notification_plugins/intro), you have to use the full image (not slim) and mount at least a custom `profiles.yaml` and a notification configuration to `/etc/crowdsec/notifications`
 
+```shell
+docker run -d \
+    -v ./profiles.yaml:/etc/crowdsec/profiles.yaml \
+    -v ./http_notification.yaml:/etc/crowdsec/notifications/http_notification.yaml \
+    -p 8080:8080 -p 6060:6060 \
+    --name crowdsec crowdsecurity/crowdsec
+```
+
 # Deployment use cases
 
 Crowdsec is composed of an `agent` that parses logs and creates `alerts`, and a

docker/config.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

docker/docker_start.sh

@@ -101,19 +101,23 @@ register_bouncer() {
 # $2 can be install, remove, upgrade
 # $3 is a list of object names separated by space
 cscli_if_clean() {
+    local itemtype="$1"
+    local action="$2"
+    local objs=$3
+    shift 3
     # loop over all objects
-    for obj in $3; do
-        if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
-            echo "Object $1/$obj is tainted, skipping"
+    for obj in $objs; do
+        if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
+            echo "Object $itemtype/$obj is tainted, skipping"
         else
 #            # Too verbose? Only show errors if not in debug mode
 #            if [ "$DEBUG" != "true" ]; then
 #                error_only=--error
 #            fi
             error_only=""
-            echo "Running: cscli $error_only $1 $2 \"$obj\""
+            echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
             # shellcheck disable=SC2086
-            cscli $error_only "$1" "$2" "$obj"
+            cscli $error_only "$itemtype" "$action" "$obj" "$@"
         fi
     done
 }
@@ -174,7 +178,7 @@ if [ ! -e "/etc/crowdsec/local_api_credentials.yaml" ] && [ ! -e "/etc/crowdsec/
         mkdir -p /etc/crowdsec/
         # if you change this, check that it still works
         # under alpine and k8s, with and without tls
-        cp -an /staging/etc/crowdsec/* /etc/crowdsec/
+        rsync -av --ignore-existing /staging/etc/crowdsec/* /etc/crowdsec
     fi
 fi
 
@@ -327,22 +331,22 @@ fi
 ## Remove collections, parsers, scenarios & postoverflows
 if [ "$DISABLE_COLLECTIONS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
+    cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force
 fi
 
 if [ "$DISABLE_PARSERS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean parsers remove "$DISABLE_PARSERS"
+    cscli_if_clean parsers remove "$DISABLE_PARSERS" --force
 fi
 
 if [ "$DISABLE_SCENARIOS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
+    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force
 fi
 
 if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
+    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
 fi
 
 ## Register bouncers via env

docker/test/tests/test_hub_collections.py

@@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor):
         cs.wait_for_log([
             # f'*collections install "{it1}"*'
             # f'*collections install "{it2}"*'
-            f'*Enabled collections : {it1}*',
-            f'*Enabled collections : {it2}*',
+            f'*Enabled collections: {it1}*',
+            f'*Enabled collections: {it2}*',
         ])
 
 
@@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor):
         assert it not in items
         logs = cs.log_lines()
         # check that there was no attempt to install
-        assert not any(f'Enabled collections : {it}' in line for line in logs)
+        assert not any(f'Enabled collections: {it}' in line for line in logs)
 
 
 # already done in bats, prividing here as example of a somewhat complex test
@@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
         # implicit check for tainted=False
         assert items[coll]['status'] == 'enabled'
         cs.wait_for_log([
-            f'*Enabled collections : {coll}*',
+            f'*Enabled collections: {coll}*',
         ])
 
         scenario = 'crowdsecurity/http-crawl-non_statics'

docker/test/tests/test_hub_scenarios.py

@@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor):
     }
     with crowdsec(flavor=flavor, environment=env) as cs:
         cs.wait_for_log([
-            f'*scenarios install "{it1}*"',
-            f'*scenarios install "{it2}*"',
+            f'*scenarios install "{it1}"*',
+            f'*scenarios install "{it2}"*',
             "*Starting processing data*"
         ])
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)

docker/test/tests/test_local_item.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+
+"""
+Test bind-mounting local items
+"""
+
+from http import HTTPStatus
+import json
+
+import pytest
+
+pytestmark = pytest.mark.docker
+
+
+def test_inject_local_item(crowdsec, tmp_path_factory, flavor):
+    """Test mounting a custom whitelist at startup"""
+
+    localitems = tmp_path_factory.mktemp('localitems')
+    custom_whitelists = localitems / 'custom_whitelists.yaml'
+
+    with open(custom_whitelists, 'w') as f:
+        f.write('{"whitelist":{"reason":"Good IPs","ip":["1.2.3.4"]}}')
+
+    volumes = {
+        custom_whitelists: {'bind': '/etc/crowdsec/parsers/s02-enrich/custom_whitelists.yaml'}
+    }
+
+    with crowdsec(flavor=flavor, volumes=volumes) as cs:
+        cs.wait_for_log([
+            "*Starting processing data*"
+        ])
+        cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
+
+        # the parser should be enabled
+        res = cs.cont.exec_run('cscli parsers list -o json')
+        assert res.exit_code == 0
+        j = json.loads(res.output)
+        items = {c['name']: c for c in j['parsers']}
+        assert items['custom_whitelists.yaml']['status'] == 'enabled,local'
+
+        # regression test: the linux collection should not be tainted
+        # (the parsers were not copied from /staging when using "cp -an" with local parsers)
+        res = cs.cont.exec_run('cscli collections inspect crowdsecurity/linux -o json')
+        assert res.exit_code == 0
+        j = json.loads(res.output)
+        # crowdsec <= 1.5.5 omits a "tainted" when it's false
+        assert j.get('tainted', False) is False

go.mod

@@ -1,15 +1,19 @@
 module github.com/crowdsecurity/crowdsec
 
-go 1.20
+go 1.21
+
+// Don't use the toolchain directive to avoid uncontrolled downloads during
+// a build, especially in sandboxed environments (freebsd, gentoo...).
+// toolchain go1.21.3
 
 require (
-	entgo.io/ent v0.11.3
-	github.com/AlecAivazis/survey/v2 v2.2.7
-	github.com/Masterminds/semver/v3 v3.1.1
-	github.com/Masterminds/sprig/v3 v3.2.2
+	entgo.io/ent v0.12.4
+	github.com/AlecAivazis/survey/v2 v2.3.7
+	github.com/Masterminds/semver/v3 v3.2.1
+	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/agext/levenshtein v1.2.1
-	github.com/alexliesenfeld/health v0.5.1
-	github.com/antonmedv/expr v1.12.5
+	github.com/alexliesenfeld/health v0.8.0
+	github.com/antonmedv/expr v1.15.3
 	github.com/appleboy/gin-jwt/v2 v2.8.0
 	github.com/aquasecurity/table v1.8.0
 	github.com/aws/aws-lambda-go v1.38.0
@@ -21,12 +25,12 @@ require (
 	github.com/c-robinson/iplib v1.0.3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.4
+	github.com/crowdsecurity/go-cs-lib v0.0.5
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dghubble/sling v1.3.0
-	github.com/docker/docker v24.0.4+incompatible
+	github.com/docker/docker v24.0.7+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/enescakir/emoji v1.0.0
 	github.com/fatih/color v1.15.0
@@ -40,11 +44,12 @@ require (
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/goccy/go-yaml v1.11.0
 	github.com/gofrs/uuid v4.0.0+incompatible
-	github.com/golang-jwt/jwt/v4 v4.4.2
+	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/go-querystring v1.0.0
 	github.com/google/uuid v1.3.0
 	github.com/google/winops v0.0.0-20230712152054-af9b550d0601
 	github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
+	github.com/gorilla/websocket v1.5.0
 	github.com/hashicorp/go-hclog v1.5.0
 	github.com/hashicorp/go-plugin v1.4.10
 	github.com/hashicorp/go-version v1.2.1
@@ -61,11 +66,11 @@ require (
 	github.com/oschwald/maxminddb-golang v1.8.0
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.14.0
-	github.com/prometheus/client_model v0.3.0
+	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/prom2json v1.3.0
 	github.com/r3labs/diff/v2 v2.14.1
-	github.com/segmentio/kafka-go v0.4.34
+	github.com/segmentio/kafka-go v0.4.45
 	github.com/shirou/gopsutil/v3 v3.23.5
 	github.com/sirupsen/logrus v1.9.3
 	github.com/slack-go/slack v0.12.2
@@ -74,21 +79,21 @@ require (
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
 	github.com/wasilibs/go-re2 v1.3.0
 	github.com/xhit/go-simple-mail/v2 v2.16.0
-	golang.org/x/crypto v0.9.0
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
+	golang.org/x/crypto v0.15.0
 	golang.org/x/mod v0.11.0
-	golang.org/x/sys v0.9.0
-	google.golang.org/grpc v1.56.1
-	google.golang.org/protobuf v1.30.0
+	golang.org/x/sys v0.14.0
+	google.golang.org/grpc v1.56.3
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/apiserver v0.27.3
+	gotest.tools/v3 v3.5.0
+	k8s.io/apiserver v0.28.4
 )
 
 require (
-	ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect
+	ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
@@ -104,12 +109,12 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.19.16 // indirect
 	github.com/go-openapi/inflect v0.19.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/loads v0.20.0 // indirect
 	github.com/go-openapi/runtime v0.19.24 // indirect
 	github.com/go-openapi/spec v0.20.0 // indirect
@@ -123,10 +128,9 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.13.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
-	github.com/huandu/xstrings v1.3.2 // indirect
+	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -140,7 +144,7 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.15.7 // indirect
+	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
@@ -163,11 +167,11 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
-	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
@@ -187,22 +191,21 @@ require (
 	github.com/zclconf/go-cty v1.8.0 // indirect
 	go.mongodb.org/mongo-driver v1.9.4 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/term v0.8.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.2.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/net v0.18.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/term v0.14.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gotest.tools/v3 v3.5.0 // indirect
-	k8s.io/api v0.27.3 // indirect
-	k8s.io/apimachinery v0.27.3 // indirect
-	k8s.io/klog/v2 v2.90.1 // indirect
-	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	k8s.io/api v0.28.4 // indirect
+	k8s.io/apimachinery v0.28.4 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )

go.sum

@@ -1,57 +1,28 @@
-ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a h1:6/nt4DODfgxzHTTg3tYy7YkVzruGQGZ/kRvXpA45KUo=
-ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a/go.mod h1:ft47uSh5hWGDCmQC9DsztZg6Xk+KagM5Ts/mZYKb9JE=
+ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 h1:JnYs/y8RJ3+MiIUp+3RgyyeO48VHLAZimqiaZYnMKk8=
+ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935/go.mod h1:isZrlzJ5cpoCoKFoY9knZug7Lq4pP1cm8g3XciLZ0Pw=
 bitbucket.org/creachadair/stringset v0.0.9 h1:L4vld9nzPt90UZNrXjNelTshD74ps4P5NGs3Iq6yN3o=
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-entgo.io/ent v0.11.3 h1:F5FBGAWiDCGder7YT+lqMnyzXl6d0xU3xMBM/SO3CMc=
-entgo.io/ent v0.11.3/go.mod h1:mvDhvynOzAsOe7anH7ynPPtMjA/eeXP96kAfweevyxc=
-github.com/AlecAivazis/survey/v2 v2.2.7 h1:5NbxkF4RSKmpywYdcRgUmos1o+roJY8duCLZXbVjoig=
-github.com/AlecAivazis/survey/v2 v2.2.7/go.mod h1:9DYvHgXtiXm6nCn+jXnOXLKbH+Yo9u8fAS/SduGdoPk=
+bitbucket.org/creachadair/stringset v0.0.9/go.mod h1:t+4WcQ4+PXTa8aQdNKe40ZP6iwesoMFWAxPGd3UGjyY=
+entgo.io/ent v0.12.4 h1:LddPnAyxls/O7DTXZvUGDj0NZIdGSu317+aoNLJWbD8=
+entgo.io/ent v0.12.4/go.mod h1:Y3JVAjtlIk8xVZYSn3t3mf8xlZIn5SAOXZQxD6kKI+Q=
+github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
+github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
-github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
-github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=
+github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
+github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
-github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw=
-github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc=
+github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s=
+github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
@@ -65,11 +36,11 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alexliesenfeld/health v0.5.1 h1:cohQdtQbJdA6bj0aMD4gdXA9xQyvh9NxWO9XLGYTYcY=
-github.com/alexliesenfeld/health v0.5.1/go.mod h1:N4NDIeQtlWumG+6z1ne1v62eQxktz5ylEgGgH9emdMw=
+github.com/alexliesenfeld/health v0.8.0 h1:lCV0i+ZJPTbqP7LfKG7p3qZBl5VhelwUFCIVWl77fgk=
+github.com/alexliesenfeld/health v0.8.0/go.mod h1:TfNP0f+9WQVWMQRzvMUjlws4ceXKEL3WR+6Hp95HUFc=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
-github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E=
-github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
+github.com/antonmedv/expr v1.15.3 h1:q3hOJZNvLvhqE8OHBs1cFRdbXFNKuA+bHmRaI+AmRmI=
+github.com/antonmedv/expr v1.15.3/go.mod h1:0E/6TxnOlRNp81GMzX9QfDPAmHo2Phg00y4JUv1ihsE=
 github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
 github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
 github.com/appleboy/gin-jwt/v2 v2.8.0 h1:Glo7cb9eBR+hj8Y7WzgfkOlqCaNLjP+RV4dNO3fpdps=
@@ -106,19 +77,11 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s
 github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
 github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU=
 github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
 github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
-github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
-github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
-github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -130,12 +93,13 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
-github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
+github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
+github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@@ -147,8 +111,8 @@ github.com/dghubble/sling v1.3.0 h1:pZHjCJq4zJvc6qVQ5wN1jo5oNZlNE0+8T/h0XeXBUKU=
 github.com/dghubble/sling v1.3.0/go.mod h1:XXShWaBWKzNLhu2OxikSNFrlsvowtz4kyRuXUG7oQKY=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.4+incompatible h1:s/LVDftw9hjblvqIeTiGYXBCD95nOEEl7qRsRrIOuQI=
-github.com/docker/docker v24.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -157,10 +121,6 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/enescakir/emoji v1.0.0 h1:W+HsNql8swfCQFtioDGDHCHri8nudlK1n5p2rHCJoog=
 github.com/enescakir/emoji v1.0.0/go.mod h1:Bt1EKuLnKDTYpLALApstIkAjdDrS/8IAgTkKp+WKFD0=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
 github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
@@ -178,20 +138,15 @@ github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-co-op/gocron v1.17.0 h1:IixLXsti+Qo0wMvmn6Kmjp2csk2ykpkcL+EmHmST18w=
 github.com/go-co-op/gocron v1.17.0/go.mod h1:IpDBSaJOVfFw7hXZuTag3SCSkqazXBBUkbQ1m1aesBs=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
-github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
-github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
@@ -227,8 +182,8 @@ github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3Hfo
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
-github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8=
-github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
 github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
@@ -283,6 +238,7 @@ github.com/go-openapi/validate v0.20.0 h1:pzutNCCBZGZlE+u8HD3JZyWdc/TVbtVwlWUp8/
 github.com/go-openapi/validate v0.20.0/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
@@ -334,51 +290,23 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
-github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
+github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
 github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
@@ -389,15 +317,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -405,8 +324,6 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/winops v0.0.0-20230712152054-af9b550d0601 h1:XvlrmqZIuwxuRE88S9mkxX+FkV+YakqbiAC5Z4OzDnM=
 github.com/google/winops v0.0.0-20230712152054-af9b550d0601/go.mod h1:rT1mcjzuvcDDbRmUTsoH6kV0DG91AkFe9UCjASraK5I=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e h1:XmA6L9IPRdUr28a+SK/oMchGgQy159wvzXA5tJ7l+40=
 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e/go.mod h1:AFIo+02s+12CEg8Gzz9kzhCbmbq6JcKNrhHffCGA9z4=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -418,18 +335,14 @@ github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQ
 github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0=
 github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl/v2 v2.13.0 h1:0Apadu1w6M11dyGFxWnmhhcMjkbAiKCv7G1r/2QgCNc=
 github.com/hashicorp/hcl/v2 v2.13.0/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M=
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
-github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ=
-github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A=
-github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
-github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw=
-github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
+github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
+github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
+github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
@@ -488,6 +401,7 @@ github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE=
 github.com/jarcoal/httpmock v1.1.0/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
+github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -495,19 +409,13 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
-github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jszwec/csvutil v1.5.1 h1:c3GFBhj6DFMUl4dMK3+B6rz2+LWWS/e9VJiVJ9t9kfQ=
 github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
@@ -517,35 +425,35 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok=
-github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
+github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
 github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
-github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI=
 github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
 github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
@@ -618,7 +526,6 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
@@ -640,8 +547,9 @@ github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
-github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
 github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
+github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -652,32 +560,21 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
-github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw=
-github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
-github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
-github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
-github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE=
-github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
 github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
 github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
 github.com/r3labs/diff/v2 v2.14.1 h1:wRZ3jB44Ny50DSXsoIcFQ27l2x+n5P31K/Pk+b9B0Ic=
@@ -690,14 +587,15 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVHUERo=
-github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0=
+github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E=
+github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y=
@@ -713,7 +611,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ=
@@ -733,7 +630,6 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -774,6 +670,7 @@ github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 h1:UFHFmFfixpmf
 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26/go.mod h1:IGhd0qMDsUa9acVjsbsT7bu3ktadtGOHI79+idTew/M=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4=
+github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo=
 github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
@@ -781,22 +678,23 @@ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq
 github.com/wasilibs/go-re2 v1.3.0 h1:LFhBNzoStM3wMie6rN2slD1cuYH2CGiHpvNL3UtcsMw=
 github.com/wasilibs/go-re2 v1.3.0/go.mod h1:AafrCXVvGRJJOImMajgJ2M7rVmWyisVK7sFshbxnVrg=
 github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
+github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
-github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw=
-github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
-github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4=
-github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xhit/go-simple-mail/v2 v2.16.0 h1:ouGy/Ww4kuaqu2E2UrDw7SvLaziWTB60ICLkIkNVccA=
 github.com/xhit/go-simple-mail/v2 v2.16.0/go.mod h1:b7P5ygho6SYE+VIqpxA6QkYfv4teeyG4MKqB3utRu98=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
 github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA=
@@ -810,11 +708,6 @@ go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S
 go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
 go.mongodb.org/mongo-driver v1.9.4 h1:qXWlnK2WCOWSxJ/Hm3XyYOGKv3ujA2btBsCyuIFvQjc=
 go.mongodb.org/mongo-driver v1.9.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -836,264 +729,155 @@ golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaE
 golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
-golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
+golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
 golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
+golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
+golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190530182044-ad28b68e88f1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
-golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE=
-golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
-golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 h1:0wxTF6pSjIIhNt7mo9GvjDfzyCOiWhmICgtO/Ah948s=
+golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1101,93 +885,21 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
-google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
+google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc=
+google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -1204,7 +916,6 @@ gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
@@ -1216,29 +927,21 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY=
 gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.27.3 h1:yR6oQXXnUEBWEWcvPWS0jQL575KoAboQPfJAuKNrw5Y=
-k8s.io/api v0.27.3/go.mod h1:C4BNvZnQOF7JA/0Xed2S+aUyJSfTGkGFxLXz9MnpIpg=
-k8s.io/apimachinery v0.27.3 h1:Ubye8oBufD04l9QnNtW05idcOe9Z3GQN8+7PqmuVcUM=
-k8s.io/apimachinery v0.27.3/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E=
-k8s.io/apiserver v0.27.3 h1:AxLvq9JYtveYWK+D/Dz/uoPCfz8JC9asR5z7+I/bbQ4=
-k8s.io/apiserver v0.27.3/go.mod h1:Y61+EaBMVWUBJtxD5//cZ48cHZbQD+yIyV/4iEBhhNA=
-k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw=
-k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY=
-k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY=
+k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0=
+k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8=
+k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg=
+k8s.io/apiserver v0.28.4 h1:BJXlaQbAU/RXYX2lRz+E1oPe3G3TKlozMMCZWu5GMgg=
+k8s.io/apiserver v0.28.4/go.mod h1:Idq71oXugKZoVGUUL2wgBCTHbUR+FYTWa4rq9j4n23w=
+k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
+k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

pkg/acquisition/acquisition.go

@@ -25,6 +25,7 @@ import (
 	kafkaacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kafka"
 	kinesisacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kinesis"
 	k8sauditacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/kubernetesaudit"
+	lokiacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki"
 	s3acquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/s3"
 	syslogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog"
 	wineventlogacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/wineventlog"
@@ -36,7 +37,7 @@ import (
 
 type DataSourceUnavailableError struct {
 	Name string
-	Err error
+	Err  error
 }
 
 func (e *DataSourceUnavailableError) Error() string {
@@ -47,7 +48,6 @@ func (e *DataSourceUnavailableError) Unwrap() error {
 	return e.Err
 }
 
-
 // The interface each datasource must implement
 type DataSource interface {
 	GetMetrics() []prometheus.Collector                                 // Returns pointers to metrics that are managed by the module
@@ -74,6 +74,7 @@ var AcquisitionSources = map[string]func() DataSource{
 	"wineventlog": func() DataSource { return &wineventlogacquisition.WinEventLogSource{} },
 	"kafka":       func() DataSource { return &kafkaacquisition.KafkaSource{} },
 	"k8s-audit":   func() DataSource { return &k8sauditacquisition.KubernetesAuditSource{} },
+	"loki":        func() DataSource { return &lokiacquisition.LokiSource{} },
 	"s3":          func() DataSource { return &s3acquisition.S3Source{} },
 }
 

pkg/acquisition/modules/cloudwatch/cloudwatch_test.go

@@ -199,7 +199,7 @@ stream_regexp: test_bad[0-9]+`),
 			},
 			expectedResLen: 0,
 		},
-		// require a group name that does exist and contains a stream in which we gonna put events
+		// require a group name that does exist and contains a stream in which we are going to put events
 		{
 			name: "group_exists_stream_exists_has_events",
 			config: []byte(`

pkg/acquisition/modules/kafka/kafka.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -37,6 +38,7 @@ type KafkaConfiguration struct {
 	Brokers                           []string   `yaml:"brokers"`
 	Topic                             string     `yaml:"topic"`
 	GroupID                           string     `yaml:"group_id"`
+	Partition                         int        `yaml:"partition"`
 	Timeout                           string     `yaml:"timeout"`
 	TLS                               *TLSConfig `yaml:"tls"`
 	configuration.DataSourceCommonCfg `yaml:",inline"`
@@ -79,12 +81,16 @@ func (k *KafkaSource) UnmarshalConfig(yamlConfig []byte) error {
 		k.Config.Mode = configuration.TAIL_MODE
 	}
 
+	k.logger.Debugf("successfully unmarshaled kafka configuration : %+v", k.Config)
+
 	return err
 }
 
 func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error {
 	k.logger = logger
 
+	k.logger.Debugf("start configuring %s source", dataSourceName)
+
 	err := k.UnmarshalConfig(yamlConfig)
 	if err != nil {
 		return err
@@ -95,7 +101,7 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error {
 		return fmt.Errorf("cannot create %s dialer: %w", dataSourceName, err)
 	}
 
-	k.Reader, err = k.Config.NewReader(dialer)
+	k.Reader, err = k.Config.NewReader(dialer, k.logger)
 	if err != nil {
 		return fmt.Errorf("cannote create %s reader: %w", dataSourceName, err)
 	}
@@ -104,6 +110,8 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error {
 		return fmt.Errorf("cannot create %s reader", dataSourceName)
 	}
 
+	k.logger.Debugf("successfully configured %s source", dataSourceName)
+
 	return nil
 }
 
@@ -143,9 +151,10 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
 	// Start processing from latest Offset
 	k.Reader.SetOffsetAt(context.Background(), time.Now())
 	for {
+		k.logger.Tracef("reading message from topic '%s'", k.Config.Topic)
 		m, err := k.Reader.ReadMessage(context.Background())
 		if err != nil {
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				return nil
 			}
 			k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err))
@@ -160,6 +169,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
 			Process: true,
 			Module:  k.GetName(),
 		}
+		k.logger.Tracef("line with message read from topic '%s': %+v", k.Config.Topic, l)
 		linesRead.With(prometheus.Labels{"topic": k.Config.Topic}).Inc()
 		var evt types.Event
 
@@ -173,6 +183,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
 }
 
 func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error {
+	k.logger.Debugf("starting %s datasource reader goroutine with configuration %+v", dataSourceName, k.Config)
 	t.Go(func() error {
 		return k.ReadMessage(out)
 	})
@@ -190,7 +201,7 @@ func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error {
 }
 
 func (k *KafkaSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
-	k.logger.Infof("start reader on topic '%s'", k.Config.Topic)
+	k.logger.Infof("start reader on brokers '%+v' with topic '%s'", k.Config.Brokers, k.Config.Topic)
 
 	t.Go(func() error {
 		defer trace.CatchPanic("crowdsec/acquis/kafka/live")
@@ -254,14 +265,23 @@ func (kc *KafkaConfiguration) NewDialer() (*kafka.Dialer, error) {
 	return dialer, nil
 }
 
-func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer) (*kafka.Reader, error) {
+func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer, logger *log.Entry) (*kafka.Reader, error) {
 	rConf := kafka.ReaderConfig{
-		Brokers: kc.Brokers,
-		Topic:   kc.Topic,
-		Dialer:  dialer,
+		Brokers:     kc.Brokers,
+		Topic:       kc.Topic,
+		Dialer:      dialer,
+		Logger:      kafka.LoggerFunc(logger.Debugf),
+		ErrorLogger: kafka.LoggerFunc(logger.Errorf),
+	}
+	if kc.GroupID != "" && kc.Partition != 0 {
+		return &kafka.Reader{}, fmt.Errorf("cannot specify both group_id and partition")
 	}
 	if kc.GroupID != "" {
 		rConf.GroupID = kc.GroupID
+	} else if kc.Partition != 0 {
+		rConf.Partition = kc.Partition
+	} else {
+		logger.Warnf("no group_id specified, crowdsec will only read from the 1st partition of the topic")
 	}
 	if err := rConf.Validate(); err != nil {
 		return &kafka.Reader{}, fmt.Errorf("while validating reader configuration: %w", err)

pkg/acquisition/modules/kafka/kafka_test.go

@@ -58,6 +58,16 @@ brokers:
 topic: crowdsec`,
 			expectedErr: "",
 		},
+		{
+			config: `
+source: kafka
+brokers:
+  - localhost:9092
+topic: crowdsec
+partition: 1
+group_id: crowdsec`,
+			expectedErr: "cannote create kafka reader: cannot specify both group_id and partition",
+		},
 	}
 
 	subLogger := log.WithFields(log.Fields{

pkg/acquisition/modules/loki/entry.go

@@ -0,0 +1,60 @@
+package loki
+
+import (
+	"encoding/json"
+	"strconv"
+	"time"
+)
+
+type Entry struct {
+	Timestamp time.Time
+	Line      string
+}
+
+func (e *Entry) UnmarshalJSON(b []byte) error {
+	var values []string
+	err := json.Unmarshal(b, &values)
+	if err != nil {
+		return err
+	}
+	t, err := strconv.Atoi(values[0])
+	if err != nil {
+		return err
+	}
+	e.Timestamp = time.Unix(int64(t), 0)
+	e.Line = values[1]
+	return nil
+}
+
+type Stream struct {
+	Stream  map[string]string `json:"stream"`
+	Entries []Entry           `json:"values"`
+}
+
+type DroppedEntry struct {
+	Labels    map[string]string `json:"labels"`
+	Timestamp time.Time         `json:"timestamp"`
+}
+
+type Tail struct {
+	Streams        []Stream       `json:"streams"`
+	DroppedEntries []DroppedEntry `json:"dropped_entries"`
+}
+
+// LokiQuery GET response.
+// See https://grafana.com/docs/loki/latest/api/#get-lokiapiv1query
+type LokiQuery struct {
+	Status string `json:"status"`
+	Data   Data   `json:"data"`
+}
+
+type Data struct {
+	ResultType string         `json:"resultType"`
+	Result     []StreamResult `json:"result"` // Warning, just stream value is handled
+	Stats      interface{}    `json:"stats"`  // Stats is boring, just ignore it
+}
+
+type StreamResult struct {
+	Stream map[string]string `json:"stream"`
+	Values []Entry           `json:"values"`
+}

pkg/acquisition/modules/loki/internal/lokiclient/loki_client.go

@@ -0,0 +1,315 @@
+package lokiclient
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/gorilla/websocket"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/tomb.v2"
+)
+
+type LokiClient struct {
+	Logger *log.Entry
+
+	config                Config
+	t                     *tomb.Tomb
+	fail_start            time.Time
+	currentTickerInterval time.Duration
+}
+
+type Config struct {
+	LokiURL    string
+	LokiPrefix string
+	Query      string
+	Headers    map[string]string
+
+	Username string
+	Password string
+
+	Since time.Duration
+	Until time.Duration
+
+	FailMaxDuration time.Duration
+
+	DelayFor int
+	Limit    int
+}
+
+func updateURI(uri string, lq LokiQueryRangeResponse, infinite bool) string {
+	u, _ := url.Parse(uri)
+	queryParams := u.Query()
+
+	if len(lq.Data.Result) > 0 {
+		lastTs := lq.Data.Result[0].Entries[len(lq.Data.Result[0].Entries)-1].Timestamp
+		// +1 the last timestamp to avoid getting the same result again.
+		queryParams.Set("start", strconv.Itoa(int(lastTs.UnixNano()+1)))
+	}
+
+	if infinite {
+		queryParams.Set("end", strconv.Itoa(int(time.Now().UnixNano())))
+	}
+
+	u.RawQuery = queryParams.Encode()
+	return u.String()
+}
+
+func (lc *LokiClient) SetTomb(t *tomb.Tomb) {
+	lc.t = t
+}
+
+func (lc *LokiClient) resetFailStart() {
+	if !lc.fail_start.IsZero() {
+		log.Infof("loki is back after %s", time.Since(lc.fail_start))
+	}
+	lc.fail_start = time.Time{}
+}
+func (lc *LokiClient) shouldRetry() bool {
+	if lc.fail_start.IsZero() {
+		lc.Logger.Warningf("loki is not available, will retry for %s", lc.config.FailMaxDuration)
+		lc.fail_start = time.Now()
+		return true
+	}
+	if time.Since(lc.fail_start) > lc.config.FailMaxDuration {
+		lc.Logger.Errorf("loki didn't manage to recover after %s, giving up", lc.config.FailMaxDuration)
+		return false
+	}
+	return true
+}
+
+func (lc *LokiClient) increaseTicker(ticker *time.Ticker) {
+	maxTicker := 10 * time.Second
+	if lc.currentTickerInterval < maxTicker {
+		lc.currentTickerInterval *= 2
+		if lc.currentTickerInterval > maxTicker {
+			lc.currentTickerInterval = maxTicker
+		}
+		ticker.Reset(lc.currentTickerInterval)
+	}
+}
+
+func (lc *LokiClient) decreaseTicker(ticker *time.Ticker) {
+	minTicker := 100 * time.Millisecond
+	if lc.currentTickerInterval != minTicker {
+		lc.currentTickerInterval = minTicker
+		ticker.Reset(lc.currentTickerInterval)
+	}
+}
+
+func (lc *LokiClient) queryRange(uri string, ctx context.Context, c chan *LokiQueryRangeResponse, infinite bool) error {
+	lc.currentTickerInterval = 100 * time.Millisecond
+	ticker := time.NewTicker(lc.currentTickerInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-lc.t.Dying():
+			return lc.t.Err()
+		case <-ticker.C:
+			resp, err := http.Get(uri)
+			if err != nil {
+				if ok := lc.shouldRetry(); !ok {
+					return errors.Wrapf(err, "error querying range")
+				} else {
+					lc.increaseTicker(ticker)
+					continue
+				}
+			}
+
+			if resp.StatusCode != http.StatusOK {
+				body, _ := io.ReadAll(resp.Body)
+				resp.Body.Close()
+				if ok := lc.shouldRetry(); !ok {
+					return errors.Wrapf(err, "bad HTTP response code: %d: %s", resp.StatusCode, string(body))
+				} else {
+					lc.increaseTicker(ticker)
+					continue
+				}
+			}
+
+			var lq LokiQueryRangeResponse
+			if err := json.NewDecoder(resp.Body).Decode(&lq); err != nil {
+				resp.Body.Close()
+				if ok := lc.shouldRetry(); !ok {
+					return errors.Wrapf(err, "error decoding Loki response")
+				} else {
+					lc.increaseTicker(ticker)
+					continue
+				}
+			}
+			resp.Body.Close()
+			lc.Logger.Tracef("Got response: %+v", lq)
+			c <- &lq
+			lc.resetFailStart()
+			if !infinite && (len(lq.Data.Result) == 0 || len(lq.Data.Result[0].Entries) < lc.config.Limit) {
+				lc.Logger.Infof("Got less than %d results (%d), stopping", lc.config.Limit, len(lq.Data.Result))
+				close(c)
+				return nil
+			}
+			if len(lq.Data.Result) > 0 {
+				lc.Logger.Debugf("(timer:%v) %d results / %d entries result[0] (uri:%s)", lc.currentTickerInterval, len(lq.Data.Result), len(lq.Data.Result[0].Entries), uri)
+			} else {
+				lc.Logger.Debugf("(timer:%v) no results (uri:%s)", lc.currentTickerInterval, uri)
+			}
+			if infinite {
+				if len(lq.Data.Result) > 0 { //as long as we get results, we keep lowest ticker
+					lc.decreaseTicker(ticker)
+				} else {
+					lc.increaseTicker(ticker)
+				}
+			}
+
+			uri = updateURI(uri, lq, infinite)
+		}
+	}
+}
+
+func (lc *LokiClient) getURLFor(endpoint string, params map[string]string) string {
+	u, err := url.Parse(lc.config.LokiURL)
+	if err != nil {
+		return ""
+	}
+	queryParams := u.Query()
+	for k, v := range params {
+		queryParams.Set(k, v)
+	}
+	u.RawQuery = queryParams.Encode()
+
+	u.Path, err = url.JoinPath(lc.config.LokiPrefix, u.Path, endpoint)
+
+	if err != nil {
+		return ""
+	}
+
+	if endpoint == "loki/api/v1/tail" {
+		if u.Scheme == "http" {
+			u.Scheme = "ws"
+		} else {
+			u.Scheme = "wss"
+		}
+	}
+
+	return u.String()
+}
+
+func (lc *LokiClient) Ready(ctx context.Context) error {
+	tick := time.NewTicker(500 * time.Millisecond)
+	url := lc.getURLFor("ready", nil)
+	for {
+		select {
+		case <-ctx.Done():
+			tick.Stop()
+			return ctx.Err()
+		case <-lc.t.Dying():
+			tick.Stop()
+			return lc.t.Err()
+		case <-tick.C:
+			lc.Logger.Debug("Checking if Loki is ready")
+			resp, err := http.Get(url)
+			if err != nil {
+				lc.Logger.Warnf("Error checking if Loki is ready: %s", err)
+				continue
+			}
+			_ = resp.Body.Close()
+			if resp.StatusCode != http.StatusOK {
+				lc.Logger.Debugf("Loki is not ready, status code: %d", resp.StatusCode)
+				continue
+			}
+			lc.Logger.Info("Loki is ready")
+			return nil
+		}
+	}
+}
+
+func (lc *LokiClient) Tail(ctx context.Context) (chan *LokiResponse, error) {
+	responseChan := make(chan *LokiResponse)
+	dialer := &websocket.Dialer{}
+	u := lc.getURLFor("loki/api/v1/tail", map[string]string{
+		"limit":     strconv.Itoa(lc.config.Limit),
+		"start":     strconv.Itoa(int(time.Now().Add(-lc.config.Since).UnixNano())),
+		"query":     lc.config.Query,
+		"delay_for": strconv.Itoa(lc.config.DelayFor),
+	})
+
+	lc.Logger.Debugf("Since: %s (%s)", lc.config.Since, time.Now().Add(-lc.config.Since))
+
+	if lc.config.Username != "" || lc.config.Password != "" {
+		dialer.Proxy = func(req *http.Request) (*url.URL, error) {
+			req.SetBasicAuth(lc.config.Username, lc.config.Password)
+			return nil, nil
+		}
+	}
+
+	requestHeader := http.Header{}
+	for k, v := range lc.config.Headers {
+		requestHeader.Add(k, v)
+	}
+	requestHeader.Set("User-Agent", "Crowdsec "+cwversion.VersionStr())
+	lc.Logger.Infof("Connecting to %s", u)
+	conn, _, err := dialer.Dial(u, requestHeader)
+
+	if err != nil {
+		lc.Logger.Errorf("Error connecting to websocket, err: %s", err)
+		return responseChan, fmt.Errorf("error connecting to websocket")
+	}
+
+	lc.t.Go(func() error {
+		for {
+			jsonResponse := &LokiResponse{}
+			err = conn.ReadJSON(jsonResponse)
+
+			if err != nil {
+				lc.Logger.Errorf("Error reading from websocket: %s", err)
+				return fmt.Errorf("websocket error: %w", err)
+			}
+
+			responseChan <- jsonResponse
+		}
+	})
+
+	return responseChan, nil
+}
+
+func (lc *LokiClient) QueryRange(ctx context.Context, infinite bool) chan *LokiQueryRangeResponse {
+	url := lc.getURLFor("loki/api/v1/query_range", map[string]string{
+		"query":     lc.config.Query,
+		"start":     strconv.Itoa(int(time.Now().Add(-lc.config.Since).UnixNano())),
+		"end":       strconv.Itoa(int(time.Now().UnixNano())),
+		"limit":     strconv.Itoa(lc.config.Limit),
+		"direction": "forward",
+	})
+
+	c := make(chan *LokiQueryRangeResponse)
+
+	lc.Logger.Debugf("Since: %s (%s)", lc.config.Since, time.Now().Add(-lc.config.Since))
+
+	requestHeader := http.Header{}
+	for k, v := range lc.config.Headers {
+		requestHeader.Add(k, v)
+	}
+
+	if lc.config.Username != "" || lc.config.Password != "" {
+		requestHeader.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(lc.config.Username+":"+lc.config.Password)))
+	}
+
+	requestHeader.Set("User-Agent", "Crowdsec "+cwversion.VersionStr())
+	lc.Logger.Infof("Connecting to %s", url)
+	lc.t.Go(func() error {
+		return lc.queryRange(url, ctx, c, infinite)
+	})
+	return c
+}
+
+func NewLokiClient(config Config) *LokiClient {
+	return &LokiClient{Logger: log.WithField("component", "lokiclient"), config: config}
+}

pkg/acquisition/modules/loki/internal/lokiclient/types.go

@@ -0,0 +1,55 @@
+package lokiclient
+
+import (
+	"encoding/json"
+	"strconv"
+	"time"
+)
+
+type Entry struct {
+	Timestamp time.Time
+	Line      string
+}
+
+func (e *Entry) UnmarshalJSON(b []byte) error {
+	var values []string
+	err := json.Unmarshal(b, &values)
+	if err != nil {
+		return err
+	}
+	t, err := strconv.Atoi(values[0])
+	if err != nil {
+		return err
+	}
+	e.Timestamp = time.Unix(0, int64(t))
+	e.Line = values[1]
+	return nil
+}
+
+type Stream struct {
+	Stream  map[string]string `json:"stream"`
+	Entries []Entry           `json:"values"`
+}
+
+type DroppedEntry struct {
+	Labels    map[string]string `json:"labels"`
+	Timestamp time.Time         `json:"timestamp"`
+}
+
+type LokiResponse struct {
+	Streams        []Stream      `json:"streams"`
+	DroppedEntries []interface{} `json:"dropped_entries"` //We don't care about the actual content i think ?
+}
+
+// LokiQuery GET response.
+// See https://grafana.com/docs/loki/latest/api/#get-lokiapiv1query
+type LokiQueryRangeResponse struct {
+	Status string `json:"status"`
+	Data   Data   `json:"data"`
+}
+
+type Data struct {
+	ResultType string      `json:"resultType"`
+	Result     []Stream    `json:"result"` // Warning, just stream value is handled
+	Stats      interface{} `json:"stats"`  // Stats is boring, just ignore it
+}

pkg/acquisition/modules/loki/loki.go

@@ -0,0 +1,370 @@
+package loki
+
+/*
+https://grafana.com/docs/loki/latest/api/#get-lokiapiv1tail
+*/
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	tomb "gopkg.in/tomb.v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	lokiclient "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki/internal/lokiclient"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+)
+
+const (
+	readyTimeout time.Duration = 3 * time.Second
+	readyLoop    int           = 3
+	readySleep   time.Duration = 10 * time.Second
+	lokiLimit    int           = 100
+)
+
+var linesRead = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "cs_lokisource_hits_total",
+		Help: "Total lines that were read.",
+	},
+	[]string{"source"})
+
+type LokiAuthConfiguration struct {
+	Username string `yaml:"username"`
+	Password string `yaml:"password"`
+}
+
+type LokiConfiguration struct {
+	URL                               string                `yaml:"url"`    // Loki url
+	Prefix                            string                `yaml:"prefix"` // Loki prefix
+	Query                             string                `yaml:"query"`  // LogQL query
+	Limit                             int                   `yaml:"limit"`  // Limit of logs to read
+	DelayFor                          time.Duration         `yaml:"delay_for"`
+	Since                             time.Duration         `yaml:"since"`
+	Headers                           map[string]string     `yaml:"headers"`        // HTTP headers for talking to Loki
+	WaitForReady                      time.Duration         `yaml:"wait_for_ready"` // Retry interval, default is 10 seconds
+	Auth                              LokiAuthConfiguration `yaml:"auth"`
+	MaxFailureDuration                time.Duration         `yaml:"max_failure_duration"` // Max duration of failure before stopping the source
+	configuration.DataSourceCommonCfg `yaml:",inline"`
+}
+
+type LokiSource struct {
+	Config LokiConfiguration
+
+	Client *lokiclient.LokiClient
+
+	logger        *log.Entry
+	lokiWebsocket string
+}
+
+func (l *LokiSource) GetMetrics() []prometheus.Collector {
+	return []prometheus.Collector{linesRead}
+}
+
+func (l *LokiSource) GetAggregMetrics() []prometheus.Collector {
+	return []prometheus.Collector{linesRead}
+}
+
+func (l *LokiSource) UnmarshalConfig(yamlConfig []byte) error {
+	err := yaml.UnmarshalStrict(yamlConfig, &l.Config)
+	if err != nil {
+		return fmt.Errorf("cannot parse loki acquisition configuration: %w", err)
+	}
+
+	if l.Config.Query == "" {
+		return errors.New("loki query is mandatory")
+	}
+
+	if l.Config.WaitForReady == 0 {
+		l.Config.WaitForReady = 10 * time.Second
+	}
+
+	if l.Config.DelayFor < 0*time.Second || l.Config.DelayFor > 5*time.Second {
+		return errors.New("delay_for should be a value between 1s and 5s")
+	}
+
+	if l.Config.Mode == "" {
+		l.Config.Mode = configuration.TAIL_MODE
+	}
+	if l.Config.Prefix == "" {
+		l.Config.Prefix = "/"
+	}
+
+	if !strings.HasSuffix(l.Config.Prefix, "/") {
+		l.Config.Prefix += "/"
+	}
+
+	if l.Config.Limit == 0 {
+		l.Config.Limit = lokiLimit
+	}
+
+	if l.Config.Mode == configuration.TAIL_MODE {
+		l.logger.Infof("Resetting since")
+		l.Config.Since = 0
+	}
+
+	if l.Config.MaxFailureDuration == 0 {
+		l.Config.MaxFailureDuration = 30 * time.Second
+	}
+
+	return nil
+}
+
+func (l *LokiSource) Configure(config []byte, logger *log.Entry) error {
+	l.Config = LokiConfiguration{}
+	l.logger = logger
+	err := l.UnmarshalConfig(config)
+	if err != nil {
+		return err
+	}
+
+	l.logger.Infof("Since value: %s", l.Config.Since.String())
+
+	clientConfig := lokiclient.Config{
+		LokiURL:         l.Config.URL,
+		Headers:         l.Config.Headers,
+		Limit:           l.Config.Limit,
+		Query:           l.Config.Query,
+		Since:           l.Config.Since,
+		Username:        l.Config.Auth.Username,
+		Password:        l.Config.Auth.Password,
+		FailMaxDuration: l.Config.MaxFailureDuration,
+	}
+
+	l.Client = lokiclient.NewLokiClient(clientConfig)
+	l.Client.Logger = logger.WithFields(log.Fields{"component": "lokiclient", "source": l.Config.URL})
+	return nil
+}
+
+func (l *LokiSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error {
+	l.logger = logger
+	l.Config = LokiConfiguration{}
+	l.Config.Mode = configuration.CAT_MODE
+	l.Config.Labels = labels
+	l.Config.UniqueId = uuid
+
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return fmt.Errorf("while parsing dsn '%s': %w", dsn, err)
+	}
+	if u.Scheme != "loki" {
+		return fmt.Errorf("invalid DSN %s for loki source, must start with loki://", dsn)
+	}
+	if u.Host == "" {
+		return errors.New("empty loki host")
+	}
+	scheme := "http"
+
+	params := u.Query()
+	if q := params.Get("ssl"); q != "" {
+		scheme = "https"
+	}
+	if q := params.Get("query"); q != "" {
+		l.Config.Query = q
+	}
+	if w := params.Get("wait_for_ready"); w != "" {
+		l.Config.WaitForReady, err = time.ParseDuration(w)
+		if err != nil {
+			return err
+		}
+	} else {
+		l.Config.WaitForReady = 10 * time.Second
+	}
+
+	if d := params.Get("delay_for"); d != "" {
+		l.Config.DelayFor, err = time.ParseDuration(d)
+		if err != nil {
+			return fmt.Errorf("invalid duration: %w", err)
+		}
+		if l.Config.DelayFor < 0*time.Second || l.Config.DelayFor > 5*time.Second {
+			return errors.New("delay_for should be a value between 1s and 5s")
+		}
+	} else {
+		l.Config.DelayFor = 0 * time.Second
+	}
+
+	if s := params.Get("since"); s != "" {
+		l.Config.Since, err = time.ParseDuration(s)
+		if err != nil {
+			return fmt.Errorf("invalid since in dsn: %w", err)
+		}
+	}
+
+	if max_failure_duration := params.Get("max_failure_duration"); max_failure_duration != "" {
+		duration, err := time.ParseDuration(max_failure_duration)
+		if err != nil {
+			return fmt.Errorf("invalid max_failure_duration in dsn: %w", err)
+		}
+		l.Config.MaxFailureDuration = duration
+	} else {
+		l.Config.MaxFailureDuration = 5 * time.Second // for OneShot mode it doesn't make sense to have longer duration
+	}
+
+	if limit := params.Get("limit"); limit != "" {
+		limit, err := strconv.Atoi(limit)
+		if err != nil {
+			return fmt.Errorf("invalid limit in dsn: %w", err)
+		}
+		l.Config.Limit = limit
+	} else {
+		l.Config.Limit = 5000 // max limit allowed by loki
+	}
+
+	if logLevel := params.Get("log_level"); logLevel != "" {
+		level, err := log.ParseLevel(logLevel)
+		if err != nil {
+			return fmt.Errorf("invalid log_level in dsn: %w", err)
+		}
+		l.Config.LogLevel = &level
+		l.logger.Logger.SetLevel(level)
+	}
+
+	l.Config.URL = fmt.Sprintf("%s://%s", scheme, u.Host)
+	if u.User != nil {
+		l.Config.Auth.Username = u.User.Username()
+		l.Config.Auth.Password, _ = u.User.Password()
+	}
+
+	clientConfig := lokiclient.Config{
+		LokiURL:  l.Config.URL,
+		Headers:  l.Config.Headers,
+		Limit:    l.Config.Limit,
+		Query:    l.Config.Query,
+		Since:    l.Config.Since,
+		Username: l.Config.Auth.Username,
+		Password: l.Config.Auth.Password,
+		DelayFor: int(l.Config.DelayFor / time.Second),
+	}
+
+	l.Client = lokiclient.NewLokiClient(clientConfig)
+	l.Client.Logger = logger.WithFields(log.Fields{"component": "lokiclient", "source": l.Config.URL})
+
+	return nil
+}
+
+func (l *LokiSource) GetMode() string {
+	return l.Config.Mode
+}
+
+func (l *LokiSource) GetName() string {
+	return "loki"
+}
+
+// OneShotAcquisition reads a set of file and returns when done
+func (l *LokiSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	l.logger.Debug("Loki one shot acquisition")
+	l.Client.SetTomb(t)
+	readyCtx, cancel := context.WithTimeout(context.Background(), l.Config.WaitForReady)
+	defer cancel()
+	err := l.Client.Ready(readyCtx)
+	if err != nil {
+		return fmt.Errorf("loki is not ready: %w", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	c := l.Client.QueryRange(ctx, false)
+
+	for {
+		select {
+		case <-t.Dying():
+			l.logger.Debug("Loki one shot acquisition stopped")
+			cancel()
+			return nil
+		case resp, ok := <-c:
+			if !ok {
+				l.logger.Info("Loki acquisition done, chan closed")
+				cancel()
+				return nil
+			}
+			for _, stream := range resp.Data.Result {
+				for _, entry := range stream.Entries {
+					l.readOneEntry(entry, l.Config.Labels, out)
+				}
+			}
+		}
+	}
+}
+
+func (l *LokiSource) readOneEntry(entry lokiclient.Entry, labels map[string]string, out chan types.Event) {
+	ll := types.Line{}
+	ll.Raw = entry.Line
+	ll.Time = entry.Timestamp
+	ll.Src = l.Config.URL
+	ll.Labels = labels
+	ll.Process = true
+	ll.Module = l.GetName()
+
+	linesRead.With(prometheus.Labels{"source": l.Config.URL}).Inc()
+	expectMode := types.LIVE
+	if l.Config.UseTimeMachine {
+		expectMode = types.TIMEMACHINE
+	}
+	out <- types.Event{
+		Line:       ll,
+		Process:    true,
+		Type:       types.LOG,
+		ExpectMode: expectMode,
+	}
+}
+
+func (l *LokiSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	l.Client.SetTomb(t)
+	readyCtx, cancel := context.WithTimeout(context.Background(), l.Config.WaitForReady)
+	defer cancel()
+	err := l.Client.Ready(readyCtx)
+	if err != nil {
+		return fmt.Errorf("loki is not ready: %w", err)
+	}
+	ll := l.logger.WithField("websocket_url", l.lokiWebsocket)
+	t.Go(func() error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		respChan := l.Client.QueryRange(ctx, true)
+		if err != nil {
+			ll.Errorf("could not start loki tail: %s", err)
+			return fmt.Errorf("while starting loki tail: %w", err)
+		}
+		for {
+			select {
+			case resp, ok := <-respChan:
+				if !ok {
+					ll.Warnf("loki channel closed")
+					return err
+				}
+				for _, stream := range resp.Data.Result {
+					for _, entry := range stream.Entries {
+						l.readOneEntry(entry, l.Config.Labels, out)
+					}
+				}
+			case <-t.Dying():
+				return nil
+			}
+		}
+	})
+	return nil
+}
+
+func (l *LokiSource) CanRun() error {
+	return nil
+}
+
+func (l *LokiSource) GetUuid() string {
+	return l.Config.UniqueId
+}
+
+func (l *LokiSource) Dump() interface{} {
+	return l
+}
+
+// SupportedModes returns the supported modes by the acquisition module
+func (l *LokiSource) SupportedModes() []string {
+	return []string{configuration.TAIL_MODE, configuration.CAT_MODE}
+}

pkg/acquisition/modules/loki/loki_test.go

@@ -0,0 +1,512 @@
+package loki_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"context"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+	tomb "gopkg.in/tomb.v2"
+	"gotest.tools/v3/assert"
+)
+
+func TestConfiguration(t *testing.T) {
+
+	log.Infof("Test 'TestConfigure'")
+
+	tests := []struct {
+		config       string
+		expectedErr  string
+		password     string
+		waitForReady time.Duration
+		delayFor     time.Duration
+		testName     string
+	}{
+		{
+			config:      `foobar: asd`,
+			expectedErr: "line 1: field foobar not found in type loki.LokiConfiguration",
+			testName:    "Unknown field",
+		},
+		{
+			config: `
+mode: tail
+source: loki`,
+			expectedErr: "loki query is mandatory",
+			testName:    "Missing url",
+		},
+		{
+			config: `
+mode: tail
+source: loki
+url: http://localhost:3100/
+`,
+			expectedErr: "loki query is mandatory",
+			testName:    "Missing query",
+		},
+		{
+			config: `
+mode: tail
+source: loki
+url: http://localhost:3100/
+query: >
+        {server="demo"}
+`,
+			expectedErr: "",
+			testName:    "Correct config",
+		},
+		{
+			config: `
+mode: tail
+source: loki
+url: http://localhost:3100/
+wait_for_ready: 5s
+query: >
+        {server="demo"}
+`,
+			expectedErr:  "",
+			testName:     "Correct config with wait_for_ready",
+			waitForReady: 5 * time.Second,
+		},
+		{
+			config: `
+mode: tail
+source: loki
+url: http://localhost:3100/
+delay_for: 1s
+query: >
+        {server="demo"}
+`,
+			expectedErr: "",
+			testName:    "Correct config with delay_for",
+			delayFor:    1 * time.Second,
+		},
+		{
+
+			config: `
+mode: tail
+source: loki
+url: http://localhost:3100/
+auth:
+  username: foo
+  password: bar
+query: >
+        {server="demo"}
+`,
+			expectedErr: "",
+			password:    "bar",
+			testName:    "Correct config with password",
+		},
+		{
+
+			config: `
+mode: tail
+source: loki
+url: http://localhost:3100/
+delay_for: 10s
+query: >
+        {server="demo"}
+`,
+			expectedErr: "delay_for should be a value between 1s and 5s",
+			testName:    "Invalid DelayFor",
+		},
+	}
+	subLogger := log.WithFields(log.Fields{
+		"type": "loki",
+	})
+	for _, test := range tests {
+		t.Run(test.testName, func(t *testing.T) {
+			lokiSource := loki.LokiSource{}
+			err := lokiSource.Configure([]byte(test.config), subLogger)
+			cstest.AssertErrorContains(t, err, test.expectedErr)
+			if test.password != "" {
+				p := lokiSource.Config.Auth.Password
+				if test.password != p {
+					t.Fatalf("Password mismatch : %s != %s", test.password, p)
+				}
+			}
+			if test.waitForReady != 0 {
+				if lokiSource.Config.WaitForReady != test.waitForReady {
+					t.Fatalf("Wrong WaitForReady %v != %v", lokiSource.Config.WaitForReady, test.waitForReady)
+				}
+			}
+			if test.delayFor != 0 {
+				if lokiSource.Config.DelayFor != test.delayFor {
+					t.Fatalf("Wrong DelayFor %v != %v", lokiSource.Config.DelayFor, test.delayFor)
+				}
+			}
+		})
+	}
+}
+
+func TestConfigureDSN(t *testing.T) {
+	log.Infof("Test 'TestConfigureDSN'")
+	tests := []struct {
+		name         string
+		dsn          string
+		expectedErr  string
+		since        time.Time
+		password     string
+		scheme       string
+		waitForReady time.Duration
+		delayFor     time.Duration
+	}{
+		{
+			name:        "Wrong scheme",
+			dsn:         "wrong://",
+			expectedErr: "invalid DSN wrong:// for loki source, must start with loki://",
+		},
+		{
+			name:        "Correct DSN",
+			dsn:         `loki://localhost:3100/?query={server="demo"}`,
+			expectedErr: "",
+		},
+		{
+			name:        "Empty host",
+			dsn:         "loki://",
+			expectedErr: "empty loki host",
+		},
+		{
+			name:        "Invalid DSN",
+			dsn:         "loki",
+			expectedErr: "invalid DSN loki for loki source, must start with loki://",
+		},
+		{
+			name:        "Invalid Delay",
+			dsn:         `loki://localhost:3100/?query={server="demo"}&delay_for=10s`,
+			expectedErr: "delay_for should be a value between 1s and 5s",
+		},
+		{
+			name:  "Bad since param",
+			dsn:   `loki://127.0.0.1:3100/?since=3h&query={server="demo"}`,
+			since: time.Now().Add(-3 * time.Hour),
+		},
+		{
+			name:     "Basic Auth",
+			dsn:      `loki://login:password@localhost:3102/?query={server="demo"}`,
+			password: "password",
+		},
+		{
+			name:         "Correct DSN",
+			dsn:          `loki://localhost:3100/?query={server="demo"}&wait_for_ready=5s&delay_for=1s`,
+			expectedErr:  "",
+			waitForReady: 5 * time.Second,
+			delayFor:     1 * time.Second,
+		},
+		{
+			name:   "SSL DSN",
+			dsn:    `loki://localhost:3100/?ssl=true`,
+			scheme: "https",
+		},
+	}
+
+	for _, test := range tests {
+		subLogger := log.WithFields(log.Fields{
+			"type": "loki",
+			"name": test.name,
+		})
+		t.Logf("Test : %s", test.name)
+		lokiSource := &loki.LokiSource{}
+		err := lokiSource.ConfigureByDSN(test.dsn, map[string]string{"type": "testtype"}, subLogger, "")
+		cstest.AssertErrorContains(t, err, test.expectedErr)
+
+		noDuration, _ := time.ParseDuration("0s")
+		if lokiSource.Config.Since != noDuration && lokiSource.Config.Since.Round(time.Second) != time.Since(test.since).Round(time.Second) {
+			t.Fatalf("Invalid since %v", lokiSource.Config.Since)
+		}
+
+		if test.password != "" {
+			p := lokiSource.Config.Auth.Password
+			if test.password != p {
+				t.Fatalf("Password mismatch : %s != %s", test.password, p)
+			}
+		}
+		if test.scheme != "" {
+			url, _ := url.Parse(lokiSource.Config.URL)
+			if test.scheme != url.Scheme {
+				t.Fatalf("Schema mismatch : %s != %s", test.scheme, url.Scheme)
+			}
+		}
+		if test.waitForReady != 0 {
+			if lokiSource.Config.WaitForReady != test.waitForReady {
+				t.Fatalf("Wrong WaitForReady %v != %v", lokiSource.Config.WaitForReady, test.waitForReady)
+			}
+		}
+		if test.delayFor != 0 {
+			if lokiSource.Config.DelayFor != test.delayFor {
+				t.Fatalf("Wrong DelayFor %v != %v", lokiSource.Config.DelayFor, test.delayFor)
+			}
+		}
+	}
+}
+
+func feedLoki(logger *log.Entry, n int, title string) error {
+	streams := LogStreams{
+		Streams: []LogStream{
+			{
+				Stream: map[string]string{
+					"server": "demo",
+					"domain": "cw.example.com",
+					"key":    title,
+				},
+				Values: make([]LogValue, n),
+			},
+		},
+	}
+	for i := 0; i < n; i++ {
+		streams.Streams[0].Values[i] = LogValue{
+			Time: time.Now(),
+			Line: fmt.Sprintf("Log line #%d %v", i, title),
+		}
+	}
+	buff, err := json.Marshal(streams)
+	if err != nil {
+		return err
+	}
+	resp, err := http.Post("http://127.0.0.1:3100/loki/api/v1/push", "application/json", bytes.NewBuffer(buff))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode != http.StatusNoContent {
+		b, _ := io.ReadAll(resp.Body)
+		logger.Error(string(b))
+		return fmt.Errorf("Bad post status %d", resp.StatusCode)
+	}
+	logger.Info(n, " Events sent")
+	return nil
+}
+
+func TestOneShotAcquisition(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on windows")
+	}
+	log.SetOutput(os.Stdout)
+	log.SetLevel(log.InfoLevel)
+	log.Info("Test 'TestStreamingAcquisition'")
+	title := time.Now().String() // Loki will be messy, with a lot of stuff, lets use a unique key
+	tests := []struct {
+		config string
+	}{
+		{
+			config: fmt.Sprintf(`
+mode: cat
+source: loki
+url: http://127.0.0.1:3100
+query: '{server="demo",key="%s"}'
+since: 1h
+`, title),
+		},
+	}
+
+	for _, ts := range tests {
+		logger := log.New()
+		subLogger := logger.WithFields(log.Fields{
+			"type": "loki",
+		})
+		lokiSource := loki.LokiSource{}
+		err := lokiSource.Configure([]byte(ts.config), subLogger)
+		if err != nil {
+			t.Fatalf("Unexpected error : %s", err)
+		}
+
+		err = feedLoki(subLogger, 20, title)
+		if err != nil {
+			t.Fatalf("Unexpected error : %s", err)
+		}
+
+		out := make(chan types.Event)
+		read := 0
+		go func() {
+			for {
+				<-out
+				read++
+			}
+		}()
+		lokiTomb := tomb.Tomb{}
+		err = lokiSource.OneShotAcquisition(out, &lokiTomb)
+		if err != nil {
+			t.Fatalf("Unexpected error : %s", err)
+		}
+		assert.Equal(t, 20, read)
+
+	}
+}
+
+func TestStreamingAcquisition(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on windows")
+	}
+	log.SetOutput(os.Stdout)
+	log.SetLevel(log.InfoLevel)
+	log.Info("Test 'TestStreamingAcquisition'")
+	title := time.Now().String()
+	tests := []struct {
+		name          string
+		config        string
+		expectedErr   string
+		streamErr     string
+		expectedLines int
+	}{
+		{
+			name: "Bad port",
+			config: `
+mode: tail
+source: loki
+url: http://127.0.0.1:3101
+query: >
+  {server="demo"}
+`, // No Loki server here
+			expectedErr:   "",
+			streamErr:     `loki is not ready: context deadline exceeded`,
+			expectedLines: 0,
+		},
+		{
+			name: "ok",
+			config: `
+mode: tail
+source: loki
+url: http://127.0.0.1:3100
+query: >
+  {server="demo"}
+`,
+			expectedErr:   "",
+			streamErr:     "",
+			expectedLines: 20,
+		},
+	}
+	for _, ts := range tests {
+		t.Run(ts.name, func(t *testing.T) {
+			logger := log.New()
+			subLogger := logger.WithFields(log.Fields{
+				"type": "loki",
+				"name": ts.name,
+			})
+
+			out := make(chan types.Event)
+			lokiTomb := tomb.Tomb{}
+			lokiSource := loki.LokiSource{}
+			err := lokiSource.Configure([]byte(ts.config), subLogger)
+			if err != nil {
+				t.Fatalf("Unexpected error : %s", err)
+			}
+			err = lokiSource.StreamingAcquisition(out, &lokiTomb)
+			cstest.AssertErrorContains(t, err, ts.streamErr)
+
+			if ts.streamErr != "" {
+				return
+			}
+
+			time.Sleep(time.Second * 2) //We need to give time to start reading from the WS
+			readTomb := tomb.Tomb{}
+			readCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+			count := 0
+
+			readTomb.Go(func() error {
+				defer cancel()
+				for {
+					select {
+					case <-readCtx.Done():
+						return readCtx.Err()
+					case evt := <-out:
+						count++
+						if !strings.HasSuffix(evt.Line.Raw, title) {
+							return fmt.Errorf("Incorrect suffix : %s", evt.Line.Raw)
+						}
+						if count == ts.expectedLines {
+							return nil
+						}
+					}
+				}
+			})
+
+			err = feedLoki(subLogger, ts.expectedLines, title)
+			if err != nil {
+				t.Fatalf("Unexpected error : %s", err)
+			}
+
+			err = readTomb.Wait()
+			cancel()
+			if err != nil {
+				t.Fatalf("Unexpected error : %s", err)
+			}
+			assert.Equal(t, count, ts.expectedLines)
+		})
+	}
+
+}
+
+func TestStopStreaming(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on windows")
+	}
+	config := `
+mode: tail
+source: loki
+url: http://127.0.0.1:3100
+query: >
+  {server="demo"}
+`
+	logger := log.New()
+	subLogger := logger.WithFields(log.Fields{
+		"type": "loki",
+	})
+	title := time.Now().String()
+	lokiSource := loki.LokiSource{}
+	err := lokiSource.Configure([]byte(config), subLogger)
+	if err != nil {
+		t.Fatalf("Unexpected error : %s", err)
+	}
+	out := make(chan types.Event)
+
+	lokiTomb := &tomb.Tomb{}
+	err = lokiSource.StreamingAcquisition(out, lokiTomb)
+	if err != nil {
+		t.Fatalf("Unexpected error : %s", err)
+	}
+	time.Sleep(time.Second * 2)
+	err = feedLoki(subLogger, 1, title)
+	if err != nil {
+		t.Fatalf("Unexpected error : %s", err)
+	}
+
+	lokiTomb.Kill(nil)
+	err = lokiTomb.Wait()
+	if err != nil {
+		t.Fatalf("Unexpected error : %s", err)
+	}
+}
+
+type LogStreams struct {
+	Streams []LogStream `json:"streams"`
+}
+
+type LogStream struct {
+	Stream map[string]string `json:"stream"`
+	Values []LogValue        `json:"values"`
+}
+
+type LogValue struct {
+	Time time.Time
+	Line string
+}
+
+func (l *LogValue) MarshalJSON() ([]byte, error) {
+	line, err := json.Marshal(l.Line)
+	if err != nil {
+		return nil, err
+	}
+	return []byte(fmt.Sprintf(`["%d",%s]`, l.Time.UnixNano(), string(line))), nil
+}

pkg/acquisition/modules/loki/timestamp.go

@@ -0,0 +1,29 @@
+package loki
+
+import (
+	"fmt"
+	"time"
+)
+
+type timestamp time.Time
+
+func (t *timestamp) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var tt time.Time
+	err := unmarshal(&tt)
+	if err == nil {
+		*t = timestamp(tt)
+		return nil
+	}
+	var d time.Duration
+	err = unmarshal(&d)
+	if err == nil {
+		*t = timestamp(time.Now().Add(-d))
+		fmt.Println("t", time.Time(*t).Format(time.RFC3339))
+		return nil
+	}
+	return err
+}
+
+func (t *timestamp) IsZero() bool {
+	return time.Time(*t).IsZero()
+}

pkg/acquisition/modules/loki/timestamp_test.go

@@ -0,0 +1,47 @@
+package loki
+
+import (
+	"testing"
+	"time"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestTimestampFail(t *testing.T) {
+	var tt timestamp
+	err := yaml.Unmarshal([]byte("plop"), tt)
+	if err == nil {
+		t.Fail()
+	}
+}
+
+func TestTimestampTime(t *testing.T) {
+	var tt timestamp
+	const ts string = "2022-06-14T12:56:39+02:00"
+	err := yaml.Unmarshal([]byte(ts), &tt)
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+	if ts != time.Time(tt).Format(time.RFC3339) {
+		t.Fail()
+	}
+}
+
+func TestTimestampDuration(t *testing.T) {
+	var tt timestamp
+	err := yaml.Unmarshal([]byte("3h"), &tt)
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+	d, err := time.ParseDuration("3h")
+	if err != nil {
+		t.Error(err)
+		t.Fail()
+	}
+	z := time.Now().Add(-d)
+	if z.Round(time.Second) != time.Time(tt).Round(time.Second) {
+		t.Fail()
+	}
+}

pkg/acquisition/modules/syslog/syslog.go

@@ -211,8 +211,10 @@ func (s *SyslogSource) handleSyslogMsg(out chan types.Event, t *tomb.Tomb, c cha
 					continue
 				}
 				line = s.buildLogFromSyslog(p2.Timestamp, p2.Hostname, p2.Tag, p2.PID, p2.Message)
+				linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc5424"}).Inc()
 			} else {
 				line = s.buildLogFromSyslog(p.Timestamp, p.Hostname, p.Tag, p.PID, p.Message)
+				linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc3164"}).Inc()
 			}
 
 			line = strings.TrimSuffix(line, "\n")

pkg/alertcontext/alertcontext.go

@@ -3,12 +3,12 @@ package alertcontext
 import (
 	"encoding/json"
 	"fmt"
+	"slices"
 	"strconv"
 
 	"github.com/antonmedv/expr"
 	"github.com/antonmedv/expr/vm"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/models"

pkg/apiserver/apic.go

@@ -15,8 +15,8 @@ import (
 	"github.com/go-openapi/strfmt"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 	"gopkg.in/tomb.v2"
+	"slices"
 
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/trace"
@@ -213,7 +213,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con
 	}
 
 	// The watcher will be authenticated by the RoundTripper the first time it will call CAPI
-	// Explicit authentication will provoke an useless supplementary call to CAPI
+	// Explicit authentication will provoke a useless supplementary call to CAPI
 	scenarios, err := ret.FetchScenariosListFromDB()
 	if err != nil {
 		return ret, fmt.Errorf("get scenario in db: %w", err)
@@ -383,19 +383,16 @@ func (a *apic) CAPIPullIsOld() (bool, error) {
 }
 
 func (a *apic) HandleDeletedDecisions(deletedDecisions []*models.Decision, delete_counters map[string]map[string]int) (int, error) {
-	var filter map[string][]string
-	var nbDeleted int
+	nbDeleted := 0
 	for _, decision := range deletedDecisions {
-		if strings.ToLower(*decision.Scope) == "ip" {
-			filter = make(map[string][]string, 1)
-			filter["value"] = []string{*decision.Value}
-		} else {
-			filter = make(map[string][]string, 3)
-			filter["value"] = []string{*decision.Value}
+		filter := map[string][]string{
+			"value":  {*decision.Value},
+			"origin": {*decision.Origin},
+		}
+		if strings.ToLower(*decision.Scope) != "ip" {
 			filter["type"] = []string{*decision.Type}
 			filter["scopes"] = []string{*decision.Scope}
 		}
-		filter["origin"] = []string{*decision.Origin}
 
 		dbCliRet, _, err := a.dbClient.SoftDeleteDecisionsWithFilter(filter)
 		if err != nil {
@@ -412,20 +409,17 @@ func (a *apic) HandleDeletedDecisions(deletedDecisions []*models.Decision, delet
 }
 
 func (a *apic) HandleDeletedDecisionsV3(deletedDecisions []*modelscapi.GetDecisionsStreamResponseDeletedItem, delete_counters map[string]map[string]int) (int, error) {
-	var filter map[string][]string
 	var nbDeleted int
 	for _, decisions := range deletedDecisions {
 		scope := decisions.Scope
 		for _, decision := range decisions.Decisions {
-			if strings.ToLower(*scope) == "ip" {
-				filter = make(map[string][]string, 1)
-				filter["value"] = []string{decision}
-			} else {
-				filter = make(map[string][]string, 2)
-				filter["value"] = []string{decision}
+			filter := map[string][]string{
+				"value":  {decision},
+				"origin": {types.CAPIOrigin},
+			}
+			if strings.ToLower(*scope) != "ip" {
 				filter["scopes"] = []string{*scope}
 			}
-			filter["origin"] = []string{types.CAPIOrigin}
 
 			dbCliRet, _, err := a.dbClient.SoftDeleteDecisionsWithFilter(filter)
 			if err != nil {
@@ -479,30 +473,42 @@ func createAlertsForDecisions(decisions []*models.Decision) []*models.Alert {
 }
 
 func createAlertForDecision(decision *models.Decision) *models.Alert {
-	newAlert := &models.Alert{}
-	newAlert.Source = &models.Source{}
-	newAlert.Source.Scope = ptr.Of("")
-	if *decision.Origin == types.CAPIOrigin { //to make things more user friendly, we replace CAPI with community-blocklist
-		newAlert.Scenario = ptr.Of(types.CAPIOrigin)
-		newAlert.Source.Scope = ptr.Of(types.CAPIOrigin)
-	} else if *decision.Origin == types.ListOrigin {
-		newAlert.Scenario = ptr.Of(*decision.Scenario)
-		newAlert.Source.Scope = ptr.Of(types.ListOrigin)
-	} else {
+	var (
+		scenario string
+		scope    string
+	)
+
+	switch *decision.Origin {
+	case types.CAPIOrigin:
+		scenario = types.CAPIOrigin
+		scope = types.CAPIOrigin
+	case types.ListOrigin:
+		scenario = *decision.Scenario
+		scope = types.ListOrigin
+	default:
+		// XXX: this or nil?
+		scenario = ""
+		scope = ""
 		log.Warningf("unknown origin %s", *decision.Origin)
 	}
-	newAlert.Message = ptr.Of("")
-	newAlert.Source.Value = ptr.Of("")
-	newAlert.StartAt = ptr.Of(time.Now().UTC().Format(time.RFC3339))
-	newAlert.StopAt = ptr.Of(time.Now().UTC().Format(time.RFC3339))
-	newAlert.Capacity = ptr.Of(int32(0))
-	newAlert.Simulated = ptr.Of(false)
-	newAlert.EventsCount = ptr.Of(int32(0))
-	newAlert.Leakspeed = ptr.Of("")
-	newAlert.ScenarioHash = ptr.Of("")
-	newAlert.ScenarioVersion = ptr.Of("")
-	newAlert.MachineID = database.CapiMachineID
-	return newAlert
+
+	return &models.Alert{
+		Source: &models.Source{
+			Scope: ptr.Of(scope),
+			Value: ptr.Of(""),
+		},
+		Scenario:        ptr.Of(scenario),
+		Message:         ptr.Of(""),
+		StartAt:         ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+		StopAt:          ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+		Capacity:        ptr.Of(int32(0)),
+		Simulated:       ptr.Of(false),
+		EventsCount:     ptr.Of(int32(0)),
+		Leakspeed:       ptr.Of(""),
+		ScenarioHash:    ptr.Of(""),
+		ScenarioVersion: ptr.Of(""),
+		MachineID:       database.CapiMachineID,
+	}
 }
 
 // This function takes in list of parent alerts and decisions and then pairs them up.

pkg/apiserver/apic_metrics.go

@@ -2,10 +2,10 @@ package apiserver
 
 import (
 	"context"
+	"slices"
 	"time"
 
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/trace"

pkg/apiserver/apic_test.go

@@ -572,7 +572,7 @@ func TestAPICWhitelists(t *testing.T) {
 					&modelscapi.GetDecisionsStreamResponseDeletedItem{
 						Decisions: []string{
 							"9.9.9.9", // This is already present in DB
-							"9.1.9.9", // This not present in DB
+							"9.1.9.9", // This is not present in DB
 						},
 						Scope: ptr.Of("Ip"),
 					}, // This is already present in DB
@@ -734,7 +734,7 @@ func TestAPICPullTop(t *testing.T) {
 					&modelscapi.GetDecisionsStreamResponseDeletedItem{
 						Decisions: []string{
 							"9.9.9.9", // This is already present in DB
-							"9.1.9.9", // This not present in DB
+							"9.1.9.9", // This is not present in DB
 						},
 						Scope: ptr.Of("Ip"),
 					}, // This is already present in DB

pkg/apiserver/jwt_test.go

@@ -55,7 +55,7 @@ func TestLogin(t *testing.T) {
 	router.ServeHTTP(w, req)
 
 	assert.Equal(t, 401, w.Code)
-	assert.Equal(t, "{\"code\":401,\"message\":\"input format error\"}", w.Body.String())
+	assert.Equal(t, "{\"code\":401,\"message\":\"validation failure list:\\npassword in body is required\"}", w.Body.String())
 
 	//Validate machine
 	err = ValidateMachine("test", config.API.Server.DbConfig)

pkg/apiserver/middlewares/v1/jwt.go

@@ -2,6 +2,7 @@ package v1
 
 import (
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -16,7 +17,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/gin-gonic/gin"
 	"github.com/go-openapi/strfmt"
-	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -46,142 +46,176 @@ func IdentityHandler(c *gin.Context) interface{} {
 	}
 }
 
-func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
-	var loginInput models.WatcherAuthRequest
-	var scenarios string
-	var err error
-	var scenariosInput []string
-	var clientMachine *ent.Machine
-	var machineID string
 
-	if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
-		if j.TlsAuth == nil {
-			c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-			c.Abort()
-			return nil, errors.New("TLS auth is not configured")
+
+type authInput struct {
+	machineID string
+	clientMachine *ent.Machine
+	scenariosInput []string
+}
+
+
+
+func (j *JWT) authTLS(c *gin.Context) (*authInput, error) {
+	ret := authInput{}
+
+	if j.TlsAuth == nil {
+		c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
+		c.Abort()
+		return nil, errors.New("TLS auth is not configured")
+	}
+
+	validCert, extractedCN, err := j.TlsAuth.ValidateCert(c)
+	if err != nil {
+		log.Error(err)
+		c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
+		c.Abort()
+		return nil, fmt.Errorf("while trying to validate client cert: %w", err)
+	}
+
+	if !validCert {
+		c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
+		c.Abort()
+		return nil, fmt.Errorf("failed cert authentication")
+	}
+
+	ret.machineID = fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
+	ret.clientMachine, err = j.DbClient.Ent.Machine.Query().
+		Where(machine.MachineId(ret.machineID)).
+		First(j.DbClient.CTX)
+	if ent.IsNotFound(err) {
+		//Machine was not found, let's create it
+		log.Infof("machine %s not found, create it", ret.machineID)
+		//let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli)
+		pwd, err := GenerateAPIKey(dummyAPIKeySize)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"ip": c.ClientIP(),
+				"cn": extractedCN,
+			}).Errorf("error generating password: %s", err)
+			return nil, fmt.Errorf("error generating password")
 		}
-		validCert, extractedCN, err := j.TlsAuth.ValidateCert(c)
+		password := strfmt.Password(pwd)
+		ret.clientMachine, err = j.DbClient.CreateMachine(&ret.machineID, &password, "", true, true, types.TlsAuthType)
 		if err != nil {
-			log.Error(err)
-			c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-			c.Abort()
-			return nil, errors.Wrap(err, "while trying to validate client cert")
+			return nil, fmt.Errorf("while creating machine entry for %s: %w", ret.machineID, err)
 		}
-		if !validCert {
-			c.JSON(http.StatusForbidden, gin.H{"message": "access forbidden"})
-			c.Abort()
-			return nil, fmt.Errorf("failed cert authentication")
+	} else if err != nil {
+		return nil, fmt.Errorf("while selecting machine entry for %s: %w", ret.machineID, err)
+	} else {
+		if ret.clientMachine.AuthType != types.TlsAuthType {
+			return nil, fmt.Errorf("machine %s attempted to auth with TLS cert but it is configured to use %s", ret.machineID, ret.clientMachine.AuthType)
 		}
+		ret.machineID = ret.clientMachine.MachineId
+	}
 
-		machineID = fmt.Sprintf("%s@%s", extractedCN, c.ClientIP())
-		clientMachine, err = j.DbClient.Ent.Machine.Query().
-			Where(machine.MachineId(machineID)).
-			First(j.DbClient.CTX)
-		if ent.IsNotFound(err) {
-			//Machine was not found, let's create it
-			log.Printf("machine %s not found, create it", machineID)
-			//let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli)
-			pwd, err := GenerateAPIKey(dummyAPIKeySize)
-			if err != nil {
-				log.WithFields(log.Fields{
-					"ip": c.ClientIP(),
-					"cn": extractedCN,
-				}).Errorf("error generating password: %s", err)
-				return nil, fmt.Errorf("error generating password")
-			}
-			password := strfmt.Password(pwd)
-			clientMachine, err = j.DbClient.CreateMachine(&machineID, &password, "", true, true, types.TlsAuthType)
-			if err != nil {
-				return "", errors.Wrapf(err, "while creating machine entry for %s", machineID)
-			}
-		} else if err != nil {
-			return "", errors.Wrapf(err, "while selecting machine entry for %s", machineID)
-		} else {
-			if clientMachine.AuthType != types.TlsAuthType {
-				return "", errors.Errorf("machine %s attempted to auth with TLS cert but it is configured to use %s", machineID, clientMachine.AuthType)
-			}
-			machineID = clientMachine.MachineId
-			loginInput := struct {
-				Scenarios []string `json:"scenarios"`
-			}{
-				Scenarios: []string{},
-			}
-			err := c.ShouldBindJSON(&loginInput)
-			if err != nil {
-				return "", errors.Wrap(err, "missing scenarios list in login request for TLS auth")
-			}
-			scenariosInput = loginInput.Scenarios
-		}
+	loginInput := struct {
+		Scenarios []string `json:"scenarios"`
+	}{
+		Scenarios: []string{},
+	}
+	err = c.ShouldBindJSON(&loginInput)
+	if err != nil {
+		return nil, fmt.Errorf("missing scenarios list in login request for TLS auth: %w", err)
+	}
+	ret.scenariosInput = loginInput.Scenarios
 
-	} else {
-		//normal auth
+	return &ret, nil
+}
 
-		if err := c.ShouldBindJSON(&loginInput); err != nil {
-			return "", errors.Wrap(err, "missing")
-		}
-		if err := loginInput.Validate(strfmt.Default); err != nil {
-			return "", errors.New("input format error")
-		}
-		machineID = *loginInput.MachineID
-		password := *loginInput.Password
-		scenariosInput = loginInput.Scenarios
 
-		clientMachine, err = j.DbClient.Ent.Machine.Query().
-			Where(machine.MachineId(machineID)).
-			First(j.DbClient.CTX)
-		if err != nil {
-			log.Printf("Error machine login for %s : %+v ", machineID, err)
-			return nil, err
-		}
 
-		if clientMachine == nil {
-			log.Errorf("Nothing for '%s'", machineID)
-			return nil, jwt.ErrFailedAuthentication
-		}
+func (j *JWT) authPlain(c *gin.Context) (*authInput, error) {
+	var loginInput models.WatcherAuthRequest
+	var err error
 
-		if clientMachine.AuthType != types.PasswordAuthType {
-			return nil, errors.Errorf("machine %s attempted to auth with password but it is configured to use %s", machineID, clientMachine.AuthType)
-		}
+	ret := authInput{}
 
-		if !clientMachine.IsValidated {
-			return nil, fmt.Errorf("machine %s not validated", machineID)
-		}
+	if err = c.ShouldBindJSON(&loginInput); err != nil {
+		return nil, fmt.Errorf("missing: %w", err)
+	}
+	if err = loginInput.Validate(strfmt.Default); err != nil {
+		return nil, err
+	}
+	ret.machineID = *loginInput.MachineID
+	password := *loginInput.Password
+	ret.scenariosInput = loginInput.Scenarios
 
-		if err = bcrypt.CompareHashAndPassword([]byte(clientMachine.Password), []byte(password)); err != nil {
-			return nil, jwt.ErrFailedAuthentication
-		}
+	ret.clientMachine, err = j.DbClient.Ent.Machine.Query().
+		Where(machine.MachineId(ret.machineID)).
+		First(j.DbClient.CTX)
+	if err != nil {
+		log.Infof("Error machine login for %s : %+v ", ret.machineID, err)
+		return nil, err
+	}
 
-		//end of normal auth
+	if ret.clientMachine == nil {
+		log.Errorf("Nothing for '%s'", ret.machineID)
+		return nil, jwt.ErrFailedAuthentication
+	}
+
+	if ret.clientMachine.AuthType != types.PasswordAuthType {
+		return nil, fmt.Errorf("machine %s attempted to auth with password but it is configured to use %s", ret.machineID, ret.clientMachine.AuthType)
+	}
+
+	if !ret.clientMachine.IsValidated {
+		return nil, fmt.Errorf("machine %s not validated", ret.machineID)
+	}
+
+	if err := bcrypt.CompareHashAndPassword([]byte(ret.clientMachine.Password), []byte(password)); err != nil {
+		return nil, jwt.ErrFailedAuthentication
 	}
 
-	if len(scenariosInput) > 0 {
-		for _, scenario := range scenariosInput {
+	return &ret, nil
+}
+
+
+func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
+	var err error
+	var auth *authInput
+
+	if c.Request.TLS != nil && len(c.Request.TLS.PeerCertificates) > 0 {
+		auth, err = j.authTLS(c)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		auth, err = j.authPlain(c)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	var scenarios string
+
+	if len(auth.scenariosInput) > 0 {
+		for _, scenario := range auth.scenariosInput {
 			if scenarios == "" {
 				scenarios = scenario
 			} else {
 				scenarios += "," + scenario
 			}
 		}
-		err = j.DbClient.UpdateMachineScenarios(scenarios, clientMachine.ID)
+		err = j.DbClient.UpdateMachineScenarios(scenarios, auth.clientMachine.ID)
 		if err != nil {
-			log.Errorf("Failed to update scenarios list for '%s': %s\n", machineID, err)
+			log.Errorf("Failed to update scenarios list for '%s': %s\n", auth.machineID, err)
 			return nil, jwt.ErrFailedAuthentication
 		}
 	}
 
-	if clientMachine.IpAddress == "" {
-		err = j.DbClient.UpdateMachineIP(c.ClientIP(), clientMachine.ID)
+	if auth.clientMachine.IpAddress == "" {
+		err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID)
 		if err != nil {
-			log.Errorf("Failed to update ip address for '%s': %s\n", machineID, err)
+			log.Errorf("Failed to update ip address for '%s': %s\n", auth.machineID, err)
 			return nil, jwt.ErrFailedAuthentication
 		}
 	}
 
-	if clientMachine.IpAddress != c.ClientIP() && clientMachine.IpAddress != "" {
-		log.Warningf("new IP address detected for machine '%s': %s (old: %s)", clientMachine.MachineId, c.ClientIP(), clientMachine.IpAddress)
-		err = j.DbClient.UpdateMachineIP(c.ClientIP(), clientMachine.ID)
+	if auth.clientMachine.IpAddress != c.ClientIP() && auth.clientMachine.IpAddress != "" {
+		log.Warningf("new IP address detected for machine '%s': %s (old: %s)", auth.clientMachine.MachineId, c.ClientIP(), auth.clientMachine.IpAddress)
+		err = j.DbClient.UpdateMachineIP(c.ClientIP(), auth.clientMachine.ID)
 		if err != nil {
-			log.Errorf("Failed to update ip address for '%s': %s\n", clientMachine.MachineId, err)
+			log.Errorf("Failed to update ip address for '%s': %s\n", auth.clientMachine.MachineId, err)
 			return nil, jwt.ErrFailedAuthentication
 		}
 	}
@@ -192,13 +226,13 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) {
 		return nil, jwt.ErrFailedAuthentication
 	}
 
-	if err := j.DbClient.UpdateMachineVersion(useragent[1], clientMachine.ID); err != nil {
-		log.Errorf("unable to update machine '%s' version '%s': %s", clientMachine.MachineId, useragent[1], err)
+	if err := j.DbClient.UpdateMachineVersion(useragent[1], auth.clientMachine.ID); err != nil {
+		log.Errorf("unable to update machine '%s' version '%s': %s", auth.clientMachine.MachineId, useragent[1], err)
 		log.Errorf("bad user agent from : %s", c.ClientIP())
 		return nil, jwt.ErrFailedAuthentication
 	}
 	return &models.WatcherAuthRequest{
-		MachineID: &machineID,
+		MachineID: &auth.machineID,
 	}, nil
 
 }

pkg/csconfig/api.go

@@ -3,7 +3,9 @@ package csconfig
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
+	"io"
 	"net"
 	"os"
 	"strings"
@@ -56,7 +58,6 @@ type CTICfg struct {
 }
 
 func (a *CTICfg) Load() error {
-
 	if a.Key == nil {
 		*a.Enabled = false
 	}
@@ -285,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
 		log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
 	}
 
-	if err := c.LoadCommon(); err != nil {
-		return fmt.Errorf("loading common configuration: %s", err)
-	}
-
 	c.API.Server.LogDir = c.Common.LogDir
 	c.API.Server.LogMedia = c.Common.LogMedia
 	c.API.Server.CompressLogs = c.Common.CompressLogs
@@ -331,43 +328,59 @@ type capiWhitelists struct {
 	Cidrs []string `yaml:"cidrs"`
 }
 
-func (s *LocalApiServerCfg) LoadCapiWhitelists() error {
-	if s.CapiWhitelistsPath == "" {
-		return nil
-	}
-	if _, err := os.Stat(s.CapiWhitelistsPath); os.IsNotExist(err) {
-		return fmt.Errorf("capi whitelist file '%s' does not exist", s.CapiWhitelistsPath)
-	}
-	fd, err := os.Open(s.CapiWhitelistsPath)
-	if err != nil {
-		return fmt.Errorf("unable to open capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
-	}
+func parseCapiWhitelists(fd io.Reader) (*CapiWhitelist, error) {
+	fromCfg := capiWhitelists{}
 
-	var fromCfg capiWhitelists
-
-	defer fd.Close()
 	decoder := yaml.NewDecoder(fd)
 	if err := decoder.Decode(&fromCfg); err != nil {
-		return fmt.Errorf("while parsing capi whitelist file '%s': %s", s.CapiWhitelistsPath, err)
+		if errors.Is(err, io.EOF) {
+			return nil, fmt.Errorf("empty file")
+		}
+		return nil, err
 	}
-	s.CapiWhitelists = &CapiWhitelist{
+	ret := &CapiWhitelist{
 		Ips:   make([]net.IP, len(fromCfg.Ips)),
 		Cidrs: make([]*net.IPNet, len(fromCfg.Cidrs)),
 	}
-	for _, v := range fromCfg.Ips {
+	for idx, v := range fromCfg.Ips {
 		ip := net.ParseIP(v)
 		if ip == nil {
-			return fmt.Errorf("unable to parse ip whitelist '%s'", v)
+			return nil, fmt.Errorf("invalid IP address: %s", v)
 		}
-		s.CapiWhitelists.Ips = append(s.CapiWhitelists.Ips, ip)
+		ret.Ips[idx] = ip
 	}
-	for _, v := range fromCfg.Cidrs {
+	for idx, v := range fromCfg.Cidrs {
 		_, tnet, err := net.ParseCIDR(v)
 		if err != nil {
-			return fmt.Errorf("unable to parse cidr whitelist '%s' : %v", v, err)
+			return nil, err
 		}
-		s.CapiWhitelists.Cidrs = append(s.CapiWhitelists.Cidrs, tnet)
+		ret.Cidrs[idx] = tnet
+	}
+
+	return ret, nil
+}
+
+func (s *LocalApiServerCfg) LoadCapiWhitelists() error {
+	if s.CapiWhitelistsPath == "" {
+		return nil
 	}
+
+	if _, err := os.Stat(s.CapiWhitelistsPath); os.IsNotExist(err) {
+		return fmt.Errorf("capi whitelist file '%s' does not exist", s.CapiWhitelistsPath)
+	}
+
+	fd, err := os.Open(s.CapiWhitelistsPath)
+	if err != nil {
+		return fmt.Errorf("while opening capi whitelist file: %s", err)
+	}
+
+	defer fd.Close()
+
+	s.CapiWhitelists, err = parseCapiWhitelists(fd)
+	if err != nil {
+		return fmt.Errorf("while parsing capi whitelist file '%s': %w", s.CapiWhitelistsPath, err)
+	}
+
 	return nil
 }
 

pkg/csconfig/api_test.go

@@ -1,14 +1,14 @@
 package csconfig
 
 import (
-	"fmt"
+	"net"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
@@ -25,7 +25,7 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
 		{
 			name: "basic valid configuration",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 			},
 			expected: &ApiCredentialsCfg{
 				URL:      "http://localhost:8080/",
@@ -36,7 +36,7 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
 		{
 			name: "invalid configuration",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/bad_lapi-secrets.yaml",
 			},
 			expected:    &ApiCredentialsCfg{},
 			expectedErr: "field unknown_key not found in type csconfig.ApiCredentialsCfg",
@@ -44,15 +44,15 @@ func TestLoadLocalApiClientCfg(t *testing.T) {
 		{
 			name: "invalid configuration filepath",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/nonexist_lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/nonexist_lapi-secrets.yaml",
 			},
 			expected:    nil,
-			expectedErr: "open ./tests/nonexist_lapi-secrets.yaml: " + cstest.FileNotFoundMessage,
+			expectedErr: "open ./testdata/nonexist_lapi-secrets.yaml: " + cstest.FileNotFoundMessage,
 		},
 		{
 			name: "valid configuration with insecure skip verify",
 			input: &LocalApiClientCfg{
-				CredentialsFilePath: "./tests/lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 				InsecureSkipVerify:  ptr.Of(false),
 			},
 			expected: &ApiCredentialsCfg{
@@ -87,7 +87,7 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 		{
 			name: "basic valid configuration",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/online-api-secrets.yaml",
+				CredentialsFilePath: "./testdata/online-api-secrets.yaml",
 			},
 			expected: &ApiCredentialsCfg{
 				URL:      "http://crowdsec.api",
@@ -98,7 +98,7 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 		{
 			name: "invalid configuration",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
+				CredentialsFilePath: "./testdata/bad_lapi-secrets.yaml",
 			},
 			expected:    &ApiCredentialsCfg{},
 			expectedErr: "failed unmarshaling api server credentials",
@@ -106,14 +106,14 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 		{
 			name: "missing field configuration",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/bad_online-api-secrets.yaml",
+				CredentialsFilePath: "./testdata/bad_online-api-secrets.yaml",
 			},
 			expected: nil,
 		},
 		{
 			name: "invalid configuration filepath",
 			input: &OnlineApiClientCfg{
-				CredentialsFilePath: "./tests/nonexist_online-api-secrets.yaml",
+				CredentialsFilePath: "./testdata/nonexist_online-api-secrets.yaml",
 			},
 			expected:    &ApiCredentialsCfg{},
 			expectedErr: "failed to read api server credentials",
@@ -136,27 +136,20 @@ func TestLoadOnlineApiClientCfg(t *testing.T) {
 
 func TestLoadAPIServer(t *testing.T) {
 	tmpLAPI := &LocalApiServerCfg{
-		ProfilesPath: "./tests/profiles.yaml",
-	}
-	if err := tmpLAPI.LoadProfiles(); err != nil {
-		t.Fatalf("loading tmp profiles: %+v", err)
+		ProfilesPath: "./testdata/profiles.yaml",
 	}
+	err := tmpLAPI.LoadProfiles()
+	require.NoError(t, err)
 
-	LogDirFullPath, err := filepath.Abs("./tests")
-	if err != nil {
-		t.Fatal(err)
-	}
 	logLevel := log.InfoLevel
 	config := &Config{}
-	fcontent, err := os.ReadFile("./tests/config.yaml")
-	if err != nil {
-		t.Fatal(err)
-	}
+	fcontent, err := os.ReadFile("./testdata/config.yaml")
+	require.NoError(t, err)
+
 	configData := os.ExpandEnv(string(fcontent))
 	err = yaml.UnmarshalStrict([]byte(configData), &config)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
+
 	tests := []struct {
 		name        string
 		input       *Config
@@ -171,18 +164,18 @@ func TestLoadAPIServer(t *testing.T) {
 					Server: &LocalApiServerCfg{
 						ListenURI: "http://crowdsec.api",
 						OnlineClient: &OnlineApiClientCfg{
-							CredentialsFilePath: "./tests/online-api-secrets.yaml",
+							CredentialsFilePath: "./testdata/online-api-secrets.yaml",
 						},
-						ProfilesPath: "./tests/profiles.yaml",
+						ProfilesPath: "./testdata/profiles.yaml",
 						PapiLogLevel: &logLevel,
 					},
 				},
 				DbConfig: &DatabaseCfg{
 					Type:   "sqlite",
-					DbPath: "./tests/test.db",
+					DbPath: "./testdata/test.db",
 				},
 				Common: &CommonCfg{
-					LogDir:   "./tests/",
+					LogDir:   "./testdata",
 					LogMedia: "stdout",
 				},
 				DisableAPI: false,
@@ -192,9 +185,10 @@ func TestLoadAPIServer(t *testing.T) {
 				ListenURI: "http://crowdsec.api",
 				TLS:       nil,
 				DbConfig: &DatabaseCfg{
-					DbPath:       "./tests/test.db",
-					Type:         "sqlite",
-					MaxOpenConns: ptr.Of(DEFAULT_MAX_OPEN_CONNS),
+					DbPath:           "./testdata/test.db",
+					Type:             "sqlite",
+					MaxOpenConns:     ptr.Of(DEFAULT_MAX_OPEN_CONNS),
+					DecisionBulkSize: defaultDecisionBulkSize,
 				},
 				ConsoleConfigPath: DefaultConfigPath("console.yaml"),
 				ConsoleConfig: &ConsoleConfig{
@@ -204,10 +198,10 @@ func TestLoadAPIServer(t *testing.T) {
 					ShareContext:          ptr.Of(false),
 					ConsoleManagement:     ptr.Of(false),
 				},
-				LogDir:   LogDirFullPath,
+				LogDir:   "./testdata",
 				LogMedia: "stdout",
 				OnlineClient: &OnlineApiClientCfg{
-					CredentialsFilePath: "./tests/online-api-secrets.yaml",
+					CredentialsFilePath: "./testdata/online-api-secrets.yaml",
 					Credentials: &ApiCredentialsCfg{
 						URL:      "http://crowdsec.api",
 						Login:    "test",
@@ -215,7 +209,7 @@ func TestLoadAPIServer(t *testing.T) {
 					},
 				},
 				Profiles:               tmpLAPI.Profiles,
-				ProfilesPath:           "./tests/profiles.yaml",
+				ProfilesPath:           "./testdata/profiles.yaml",
 				UseForwardedForHeaders: false,
 				PapiLogLevel:           &logLevel,
 			},
@@ -228,34 +222,91 @@ func TestLoadAPIServer(t *testing.T) {
 					Server: &LocalApiServerCfg{},
 				},
 				Common: &CommonCfg{
-					LogDir:   "./tests/",
+					LogDir:   "./testdata/",
 					LogMedia: "stdout",
 				},
 				DisableAPI: false,
 			},
 			expected: &LocalApiServerCfg{
-				Enable:    ptr.Of(true),
+				Enable:       ptr.Of(true),
 				PapiLogLevel: &logLevel,
 			},
 			expectedErr: "no database configuration provided",
 		},
 	}
 
-	for idx, test := range tests {
-		err := test.input.LoadAPIServer()
-		if err == nil && test.expectedErr != "" {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
-			t.Fatalf("Test number %d/%d expected error, didn't get it", idx+1, len(tests))
-		} else if test.expectedErr != "" {
-			fmt.Printf("ERR: %+v\n", err)
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.expectedErr) {
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.expectedErr,
-					fmt.Sprintf("%s", err))
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.input.LoadAPIServer()
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
+			if tc.expectedErr != "" {
+				return
+			}
+
+			assert.Equal(t, tc.expected, tc.input.API.Server)
+		})
+	}
+}
+
+func mustParseCIDRNet(t *testing.T, s string) *net.IPNet {
+	_, ipNet, err := net.ParseCIDR(s)
+	require.NoError(t, err)
+	return ipNet
+}
+
+func TestParseCapiWhitelists(t *testing.T) {
+	tests := []struct {
+		name        string
+		input       string
+		expected    *CapiWhitelist
+		expectedErr string
+	}{
+		{
+			name:  "empty file",
+			input: "",
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{},
+				Cidrs: []*net.IPNet{},
+			},
+			expectedErr: "empty file",
+		},
+		{
+			name:  "empty ip and cidr",
+			input: `{"ips": [], "cidrs": []}`,
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{},
+				Cidrs: []*net.IPNet{},
+			},
+		},
+		{
+			name:  "some ip",
+			input: `{"ips": ["1.2.3.4"]}`,
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{net.IPv4(1, 2, 3, 4)},
+				Cidrs: []*net.IPNet{},
+			},
+		},
+		{
+			name:  "some cidr",
+			input: `{"cidrs": ["1.2.3.0/24"]}`,
+			expected: &CapiWhitelist{
+				Ips:   []net.IP{},
+				Cidrs: []*net.IPNet{mustParseCIDRNet(t, "1.2.3.0/24")},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			wl, err := parseCapiWhitelists(strings.NewReader(tc.input))
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
+			if tc.expectedErr != "" {
+				return
 			}
 
-			assert.Equal(t, test.expected, test.input.API.Server)
-		}
+			assert.Equal(t, tc.expected, wl)
+		})
 	}
 }

pkg/csconfig/common.go

@@ -14,7 +14,7 @@ type CommonCfg struct {
 	LogMedia       string     `yaml:"log_media"`
 	LogDir         string     `yaml:"log_dir,omitempty"` //if LogMedia = file
 	LogLevel       *log.Level `yaml:"log_level"`
-	WorkingDir     string     `yaml:"working_dir,omitempty"` ///var/run
+	WorkingDir     string     `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
 	CompressLogs   *bool      `yaml:"compress_logs,omitempty"`
 	LogMaxSize     int        `yaml:"log_max_size,omitempty"`
 	LogMaxAge      int        `yaml:"log_max_age,omitempty"`
@@ -22,15 +22,18 @@ type CommonCfg struct {
 	ForceColorLogs bool       `yaml:"force_color_logs,omitempty"`
 }
 
-func (c *Config) LoadCommon() error {
+func (c *Config) loadCommon() error {
 	var err error
 	if c.Common == nil {
-		return fmt.Errorf("no common block provided in configuration file")
+		c.Common = &CommonCfg{}
+	}
+
+	if c.Common.LogMedia == "" {
+		c.Common.LogMedia = "stdout"
 	}
 
 	var CommonCleanup = []*string{
 		&c.Common.LogDir,
-		&c.Common.WorkingDir,
 	}
 	for _, k := range CommonCleanup {
 		if *k == "" {

pkg/csconfig/common_test.go

@@ -1,94 +0,0 @@
-package csconfig
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestLoadCommon(t *testing.T) {
-	pidDirPath := "./tests"
-	LogDirFullPath, err := filepath.Abs("./tests/log/")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	WorkingDirFullPath, err := filepath.Abs("./tests")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	tests := []struct {
-		name           string
-		Input          *Config
-		expectedResult *CommonCfg
-		err            string
-	}{
-		{
-			name: "basic valid configuration",
-			Input: &Config{
-				Common: &CommonCfg{
-					Daemonize:  true,
-					PidDir:     "./tests",
-					LogMedia:   "file",
-					LogDir:     "./tests/log/",
-					WorkingDir: "./tests/",
-				},
-			},
-			expectedResult: &CommonCfg{
-				Daemonize:  true,
-				PidDir:     pidDirPath,
-				LogMedia:   "file",
-				LogDir:     LogDirFullPath,
-				WorkingDir: WorkingDirFullPath,
-			},
-		},
-		{
-			name: "empty working dir",
-			Input: &Config{
-				Common: &CommonCfg{
-					Daemonize: true,
-					PidDir:    "./tests",
-					LogMedia:  "file",
-					LogDir:    "./tests/log/",
-				},
-			},
-			expectedResult: &CommonCfg{
-				Daemonize: true,
-				PidDir:    pidDirPath,
-				LogMedia:  "file",
-				LogDir:    LogDirFullPath,
-			},
-		},
-		{
-			name:           "no common",
-			Input:          &Config{},
-			expectedResult: nil,
-		},
-	}
-
-	for idx, test := range tests {
-		err := test.Input.LoadCommon()
-		if err == nil && test.err != "" {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
-		} else if test.err != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
-			}
-		}
-
-		isOk := assert.Equal(t, test.expectedResult, test.Input.Common)
-		if !isOk {
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
-		}
-	}
-}

pkg/csconfig/config.go

@@ -1,3 +1,5 @@
+// Package csconfig contains the configuration structures for crowdsec and cscli.
+
 package csconfig
 
 import (
@@ -21,7 +23,7 @@ var defaultDataDir = "/var/lib/crowdsec/data/"
 
 // Config contains top-level defaults -> overridden by configuration file -> overridden by CLI flags
 type Config struct {
-	//just a path to ourself :p
+	//just a path to ourselves :p
 	FilePath     *string             `yaml:"-"`
 	Self         []byte              `yaml:"-"`
 	Common       *CommonCfg          `yaml:"common,omitempty"`
@@ -34,16 +36,7 @@ type Config struct {
 	PluginConfig *PluginCfg          `yaml:"plugin_config,omitempty"`
 	DisableAPI   bool                `yaml:"-"`
 	DisableAgent bool                `yaml:"-"`
-	Hub          *Hub                `yaml:"-"`
-}
-
-func (c *Config) Dump() error {
-	out, err := yaml.Marshal(c)
-	if err != nil {
-		return fmt.Errorf("failed marshaling config: %w", err)
-	}
-	fmt.Printf("%s", string(out))
-	return nil
+	Hub          *LocalHubCfg        `yaml:"-"`
 }
 
 func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
@@ -65,6 +58,37 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
 		// this is actually the "merged" yaml
 		return nil, "", fmt.Errorf("%s: %w", configFile, err)
 	}
+
+	if cfg.Prometheus == nil {
+		cfg.Prometheus = &PrometheusCfg{}
+	}
+
+	if cfg.Prometheus.ListenAddr == "" {
+		cfg.Prometheus.ListenAddr = "127.0.0.1"
+		log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr)
+	}
+
+	if cfg.Prometheus.ListenPort == 0 {
+		cfg.Prometheus.ListenPort = 6060
+		log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
+	}
+
+	if err = cfg.loadCommon(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadConfigurationPaths(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadHub(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadCSCLI(); err != nil {
+		return nil, "", err
+	}
+
 	return &cfg, configData, nil
 }
 
@@ -72,11 +96,8 @@ func NewDefaultConfig() *Config {
 	logLevel := log.InfoLevel
 	commonCfg := CommonCfg{
 		Daemonize: false,
-		PidDir:    "/tmp/",
 		LogMedia:  "stdout",
-		//LogDir unneeded
-		LogLevel:   &logLevel,
-		WorkingDir: ".",
+		LogLevel:  &logLevel,
 	}
 	prometheus := PrometheusCfg{
 		Enabled: true,

pkg/csconfig/config_paths.go

@@ -15,7 +15,7 @@ type ConfigurationPaths struct {
 	NotificationDir    string `yaml:"notification_dir,omitempty"`
 }
 
-func (c *Config) LoadConfigurationPaths() error {
+func (c *Config) loadConfigurationPaths() error {
 	var err error
 	if c.ConfigPaths == nil {
 		return fmt.Errorf("no configuration paths provided")

pkg/csconfig/config_test.go

@@ -5,42 +5,43 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestNormalLoad(t *testing.T) {
-	_, _, err := NewConfig("./tests/config.yaml", false, false, false)
+	_, _, err := NewConfig("./testdata/config.yaml", false, false, false)
 	require.NoError(t, err)
 
-	_, _, err = NewConfig("./tests/xxx.yaml", false, false, false)
-	assert.EqualError(t, err, "while reading yaml file: open ./tests/xxx.yaml: "+cstest.FileNotFoundMessage)
+	_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
+	require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
 
-	_, _, err = NewConfig("./tests/simulation.yaml", false, false, false)
-	assert.EqualError(t, err, "./tests/simulation.yaml: yaml: unmarshal errors:\n  line 1: field simulation not found in type csconfig.Config")
+	_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
+	require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n  line 1: field simulation not found in type csconfig.Config")
 }
 
 func TestNewCrowdSecConfig(t *testing.T) {
 	tests := []struct {
-		name           string
-		expectedResult *Config
+		name     string
+		expected *Config
 	}{
 		{
-			name:           "new configuration: basic",
-			expectedResult: &Config{},
+			name:     "new configuration: basic",
+			expected: &Config{},
 		},
 	}
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
 			result := &Config{}
-			assert.Equal(t, tc.expectedResult, result)
+			assert.Equal(t, tc.expected, result)
 		})
 	}
 }
 
 func TestDefaultConfig(t *testing.T) {
 	x := NewDefaultConfig()
-	err := x.Dump()
-	require.NoError(t, err)
+	_, err := yaml.Marshal(x)
+	require.NoError(t, err, "failed marshaling config: %s", err)
 }

pkg/csconfig/console.go

@@ -82,23 +82,3 @@ func (c *LocalApiServerCfg) LoadConsoleConfig() error {
 
 	return nil
 }
-
-func (c *LocalApiServerCfg) DumpConsoleConfig() error {
-	var out []byte
-	var err error
-
-	if out, err = yaml.Marshal(c.ConsoleConfig); err != nil {
-		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
-	}
-	if c.ConsoleConfigPath == "" {
-		c.ConsoleConfigPath = DefaultConsoleConfigFilePath
-		log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
-
-	}
-
-	if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
-		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
-	}
-
-	return nil
-}

pkg/csconfig/crowdsec_service.go

@@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct {
 	BucketStateDumpDir        string            `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
 	BucketsGCEnabled          bool              `yaml:"-"`                          // we need to garbage collect buckets when in forensic mode
 
-	HubDir             string              `yaml:"-"`
-	DataDir            string              `yaml:"-"`
-	ConfigDir          string              `yaml:"-"`
-	HubIndexFile       string              `yaml:"-"`
 	SimulationFilePath string              `yaml:"-"`
 	ContextToSend      map[string][]string `yaml:"-"`
 }
@@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("load error (simulation): %w", err)
 	}
 
-	c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
-	c.Crowdsec.DataDir = c.ConfigPaths.DataDir
-	c.Crowdsec.HubDir = c.ConfigPaths.HubDir
-	c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
-
 	if c.Crowdsec.ParserRoutinesCount <= 0 {
 		c.Crowdsec.ParserRoutinesCount = 1
 	}
@@ -145,15 +136,11 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("loading api client: %s", err)
 	}
 
-	if err := c.LoadHub(); err != nil {
-		return fmt.Errorf("while loading hub: %w", err)
-	}
-
 	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
 	fallback := false
 	if c.Crowdsec.ConsoleContextPath == "" {
 		// fallback to default config file
-		c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
+		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
 		fallback = true
 	}
 

pkg/csconfig/crowdsec_service_test.go

@@ -1,82 +1,65 @@
 package csconfig
 
 import (
-	"fmt"
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 	"github.com/crowdsecurity/go-cs-lib/ptr"
-
-	"github.com/stretchr/testify/require"
 )
 
 func TestLoadCrowdsec(t *testing.T) {
-	acquisFullPath, err := filepath.Abs("./tests/acquis.yaml")
-	require.NoError(t, err)
-
-	acquisInDirFullPath, err := filepath.Abs("./tests/acquis/acquis.yaml")
-	require.NoError(t, err)
-
-	acquisDirFullPath, err := filepath.Abs("./tests/acquis")
-	require.NoError(t, err)
-
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
+	acquisFullPath, err := filepath.Abs("./testdata/acquis.yaml")
 	require.NoError(t, err)
 
-	configDirFullPath, err := filepath.Abs("./tests")
+	acquisInDirFullPath, err := filepath.Abs("./testdata/acquis/acquis.yaml")
 	require.NoError(t, err)
 
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
+	acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
 	require.NoError(t, err)
 
-	contextFileFullPath, err := filepath.Abs("./tests/context.yaml")
+	contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
 	require.NoError(t, err)
 
 	tests := []struct {
-		name           string
-		input          *Config
-		expectedResult *CrowdsecServiceCfg
-		expectedErr    string
+		name        string
+		input       *Config
+		expected    *CrowdsecServiceCfg
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					AcquisitionFilePath:       "./tests/acquis.yaml",
-					SimulationFilePath:        "./tests/simulation.yaml",
-					ConsoleContextPath:        "./tests/context.yaml",
+					AcquisitionFilePath:       "./testdata/acquis.yaml",
+					SimulationFilePath:        "./testdata/simulation.yaml",
+					ConsoleContextPath:        "./testdata/context.yaml",
 					ConsoleContextValueLength: 2500,
 				},
 			},
-			expectedResult: &CrowdsecServiceCfg{
+			expected: &CrowdsecServiceCfg{
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				ConsoleContextPath:        contextFileFullPath,
 				AcquisitionFilePath:       acquisFullPath,
-				ConfigDir:                 configDirFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
 				ConsoleContextValueLength: 2500,
 				AcquisitionFiles:          []string{acquisFullPath},
-				SimulationFilePath:        "./tests/simulation.yaml",
+				SimulationFilePath:        "./testdata/simulation.yaml",
 				ContextToSend: map[string][]string{
 					"source_ip": {"evt.Parsed.source_ip"},
 				},
@@ -89,31 +72,27 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "basic valid configuration with acquisition dir",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					AcquisitionFilePath: "./tests/acquis.yaml",
-					AcquisitionDirPath:  "./tests/acquis/",
-					SimulationFilePath:  "./tests/simulation.yaml",
-					ConsoleContextPath:  "./tests/context.yaml",
+					AcquisitionFilePath: "./testdata/acquis.yaml",
+					AcquisitionDirPath:  "./testdata/acquis/",
+					SimulationFilePath:  "./testdata/simulation.yaml",
+					ConsoleContextPath:  "./testdata/context.yaml",
 				},
 			},
-			expectedResult: &CrowdsecServiceCfg{
+			expected: &CrowdsecServiceCfg{
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        acquisDirFullPath,
 				AcquisitionFilePath:       acquisFullPath,
 				ConsoleContextPath:        contextFileFullPath,
-				ConfigDir:                 configDirFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -122,7 +101,7 @@ func TestLoadCrowdsec(t *testing.T) {
 				ContextToSend: map[string][]string{
 					"source_ip": {"evt.Parsed.source_ip"},
 				},
-				SimulationFilePath: "./tests/simulation.yaml",
+				SimulationFilePath: "./testdata/simulation.yaml",
 				SimulationConfig: &SimulationConfig{
 					Simulation: ptr.Of(false),
 				},
@@ -132,28 +111,24 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "no acquisition file and dir",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					ConsoleContextPath:        contextFileFullPath,
+					ConsoleContextPath:        "./testdata/context.yaml",
 					ConsoleContextValueLength: 10,
 				},
 			},
-			expectedResult: &CrowdsecServiceCfg{
+			expected: &CrowdsecServiceCfg{
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				AcquisitionFilePath:       "",
-				ConfigDir:                 configDirFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
 				ConsoleContextPath:        contextFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
@@ -173,18 +148,18 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "non existing acquisition file",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 				API: &APICfg{
 					Client: &LocalApiClientCfg{
-						CredentialsFilePath: "./tests/lapi-secrets.yaml",
+						CredentialsFilePath: "./testdata/lapi-secrets.yaml",
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
 					ConsoleContextPath:  "",
-					AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
+					AcquisitionFilePath: "./testdata/acquis_not_exist.yaml",
 				},
 			},
 			expectedErr: cstest.FileNotFoundMessage,
@@ -193,26 +168,25 @@ func TestLoadCrowdsec(t *testing.T) {
 			name: "agent disabled",
 			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir: "./tests",
+					ConfigDir: "./testdata",
 					DataDir:   "./data",
 					HubDir:    "./hub",
 				},
 			},
-			expectedResult: nil,
+			expected: nil,
 		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			fmt.Printf("TEST '%s'\n", tc.name)
 			err := tc.input.LoadCrowdsec()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return
 			}
 
-			require.Equal(t, tc.expectedResult, tc.input.Crowdsec)
+			require.Equal(t, tc.expected, tc.input.Crowdsec)
 		})
 	}
 }

pkg/csconfig/cscli.go

@@ -1,5 +1,9 @@
 package csconfig
 
+import (
+	"fmt"
+)
+
 /*cscli specific config, such as hub directory*/
 type CscliCfg struct {
 	Output             string            `yaml:"output,omitempty"`
@@ -7,25 +11,19 @@ type CscliCfg struct {
 	HubBranch          string            `yaml:"hub_branch"`
 	SimulationConfig   *SimulationConfig `yaml:"-"`
 	DbConfig           *DatabaseCfg      `yaml:"-"`
-	HubDir             string            `yaml:"-"`
-	DataDir            string            `yaml:"-"`
-	ConfigDir          string            `yaml:"-"`
-	HubIndexFile       string            `yaml:"-"`
+
 	SimulationFilePath string            `yaml:"-"`
 	PrometheusUrl      string            `yaml:"prometheus_uri"`
 }
 
-func (c *Config) LoadCSCLI() error {
+func (c *Config) loadCSCLI() error {
 	if c.Cscli == nil {
 		c.Cscli = &CscliCfg{}
 	}
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
+
+	if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
+		c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
 	}
-	c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
-	c.Cscli.DataDir = c.ConfigPaths.DataDir
-	c.Cscli.HubDir = c.ConfigPaths.HubDir
-	c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
 
 	return nil
 }

pkg/csconfig/cscli_test.go

@@ -1,84 +1,52 @@
 package csconfig
 
 import (
-	"fmt"
-	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadCSCLI(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	dataFullPath, err := filepath.Abs("./data")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	configDirFullPath, err := filepath.Abs("./tests")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	tests := []struct {
-		name           string
-		Input          *Config
-		expectedResult *CscliCfg
-		err            string
+		name        string
+		input       *Config
+		expected    *CscliCfg
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./tests",
+					ConfigDir:    "./testdata",
 					DataDir:      "./data",
 					HubDir:       "./hub",
 					HubIndexFile: "./hub/.index.json",
 				},
+				Prometheus: &PrometheusCfg{
+					Enabled:    true,
+					Level:      "full",
+					ListenAddr: "127.0.0.1",
+					ListenPort: 6060,
+				},
 			},
-			expectedResult: &CscliCfg{
-				ConfigDir:    configDirFullPath,
-				DataDir:      dataFullPath,
-				HubDir:       hubFullPath,
-				HubIndexFile: hubIndexFileFullPath,
+			expected: &CscliCfg{
+				PrometheusUrl: "http://127.0.0.1:6060/metrics",
 			},
 		},
-		{
-			name:           "no configuration path",
-			Input:          &Config{},
-			expectedResult: &CscliCfg{},
-		},
 	}
 
-	for idx, test := range tests {
-		err := test.Input.LoadCSCLI()
-		if err == nil && test.err != "" {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
-		} else if test.err != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.input.loadCSCLI()
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
+			if tc.expectedErr != "" {
+				return
 			}
-		}
 
-		isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli)
-		if !isOk {
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
-		}
+			assert.Equal(t, tc.expected, tc.input.Cscli)
+		})
 	}
 }

pkg/csconfig/database.go

@@ -10,13 +10,12 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 )
 
-var DEFAULT_MAX_OPEN_CONNS = 100
-
 const (
+	DEFAULT_MAX_OPEN_CONNS  = 100
 	defaultDecisionBulkSize = 1000
 	// we need an upper bound due to the sqlite limit of 32k variables in a query
 	// we have 15 variables per decision, so 32768/15 = 2184.5333
-	maxDecisionBulkSize     = 2000
+	maxDecisionBulkSize = 2000
 )
 
 type DatabaseCfg struct {

pkg/csconfig/database_test.go

@@ -1,28 +1,27 @@
 package csconfig
 
 import (
-	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 )
 
 func TestLoadDBConfig(t *testing.T) {
 	tests := []struct {
-		name           string
-		Input          *Config
-		expectedResult *DatabaseCfg
-		err            string
+		name        string
+		input       *Config
+		expected    *DatabaseCfg
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				DbConfig: &DatabaseCfg{
 					Type:         "sqlite",
-					DbPath:       "./tests/test.db",
+					DbPath:       "./testdata/test.db",
 					MaxOpenConns: ptr.Of(10),
 				},
 				Cscli: &CscliCfg{},
@@ -30,38 +29,31 @@ func TestLoadDBConfig(t *testing.T) {
 					Server: &LocalApiServerCfg{},
 				},
 			},
-			expectedResult: &DatabaseCfg{
-				Type:         "sqlite",
-				DbPath:       "./tests/test.db",
-				MaxOpenConns: ptr.Of(10),
+			expected: &DatabaseCfg{
+				Type:             "sqlite",
+				DbPath:           "./testdata/test.db",
+				MaxOpenConns:     ptr.Of(10),
 				DecisionBulkSize: defaultDecisionBulkSize,
 			},
 		},
 		{
-			name:           "no configuration path",
-			Input:          &Config{},
-			expectedResult: nil,
+			name:        "no configuration path",
+			input:       &Config{},
+			expected:    nil,
+			expectedErr: "no database configuration provided",
 		},
 	}
 
-	for idx, test := range tests {
-		err := test.Input.LoadDBConfig()
-		if err == nil && test.err != "" {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
-		} else if test.err != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.input.LoadDBConfig()
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
+			if tc.expectedErr != "" {
+				return
 			}
-		}
-		isOk := assert.Equal(t, test.expectedResult, test.Input.DbConfig)
-		if !isOk {
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
-		}
+
+			assert.Equal(t, tc.expected, tc.input.DbConfig)
+		})
 	}
 }

pkg/csconfig/fflag.go

@@ -10,7 +10,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
-
 // LoadFeatureFlagsEnv parses the environment variables to enable feature flags.
 func LoadFeatureFlagsEnv(logger *log.Logger) error {
 	if err := fflag.Crowdsec.SetFromEnv(logger); err != nil {
@@ -19,13 +18,18 @@ func LoadFeatureFlagsEnv(logger *log.Logger) error {
 	return nil
 }
 
-
-// LoadFeatureFlags parses feature.yaml to enable feature flags.
+// FeatureFlagsFileLocation returns the path to the feature.yaml file.
 // The file is in the same directory as config.yaml, which is provided
 // as the fist parameter. This can be different than ConfigPaths.ConfigDir
-func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
+// because we have not read config.yaml yet so we don't know the value of ConfigDir.
+func GetFeatureFilePath(configPath string) string {
 	dir := filepath.Dir(configPath)
-	featurePath := filepath.Join(dir, "feature.yaml")
+	return filepath.Join(dir, "feature.yaml")
+}
+
+// LoadFeatureFlags parses feature.yaml to enable feature flags.
+func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
+	featurePath := GetFeatureFilePath(configPath)
 
 	if err := fflag.Crowdsec.SetFromYamlFile(featurePath, logger); err != nil {
 		return fmt.Errorf("file %s: %s", featurePath, err)
@@ -33,7 +37,6 @@ func LoadFeatureFlagsFile(configPath string, logger *log.Logger) error {
 	return nil
 }
 
-
 // ListFeatureFlags returns a list of the enabled feature flags.
 func ListFeatureFlags() string {
 	enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()

pkg/csconfig/hub.go

@@ -1,23 +1,19 @@
 package csconfig
 
-/*cscli specific config, such as hub directory*/
-type Hub struct {
-	HubDir       string `yaml:"-"`
-	ConfigDir    string `yaml:"-"`
-	HubIndexFile string `yaml:"-"`
-	DataDir      string `yaml:"-"`
+// LocalHubCfg holds the configuration for a local hub: where to download etc.
+type LocalHubCfg struct {
+	HubIndexFile   string	// Path to the local index file
+	HubDir         string	// Where the hub items are downloaded
+	InstallDir     string	// Where to install items
+	InstallDataDir string	// Where to install data
 }
 
-func (c *Config) LoadHub() error {
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
-
-	c.Hub = &Hub{
-		HubIndexFile: c.ConfigPaths.HubIndexFile,
-		ConfigDir:    c.ConfigPaths.ConfigDir,
-		HubDir:       c.ConfigPaths.HubDir,
-		DataDir:      c.ConfigPaths.DataDir,
+func (c *Config) loadHub() error {
+	c.Hub = &LocalHubCfg{
+		HubIndexFile:   c.ConfigPaths.HubIndexFile,
+		HubDir:         c.ConfigPaths.HubDir,
+		InstallDir:     c.ConfigPaths.ConfigDir,
+		InstallDataDir: c.ConfigPaths.DataDir,
 	}
 
 	return nil

pkg/csconfig/hub_test.go

@@ -1,94 +1,49 @@
 package csconfig
 
 import (
-	"fmt"
-	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadHub(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	dataFullPath, err := filepath.Abs("./data")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	configDirFullPath, err := filepath.Abs("./tests")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	tests := []struct {
-		name           string
-		Input          *Config
-		expectedResult *Hub
-		err            string
+		name        string
+		input       *Config
+		expected    *LocalHubCfg
+		expectedErr string
 	}{
 		{
 			name: "basic valid configuration",
-			Input: &Config{
+			input: &Config{
 				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./tests",
+					ConfigDir:    "./testdata",
 					DataDir:      "./data",
 					HubDir:       "./hub",
 					HubIndexFile: "./hub/.index.json",
 				},
 			},
-			expectedResult: &Hub{
-				ConfigDir:    configDirFullPath,
-				DataDir:      dataFullPath,
-				HubDir:       hubFullPath,
-				HubIndexFile: hubIndexFileFullPath,
+			expected: &LocalHubCfg{
+				HubDir:         "./hub",
+				HubIndexFile:   "./hub/.index.json",
+				InstallDir:     "./testdata",
+				InstallDataDir: "./data",
 			},
 		},
-		{
-			name: "no data dir",
-			Input: &Config{
-				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./tests",
-					HubDir:       "./hub",
-					HubIndexFile: "./hub/.index.json",
-				},
-			},
-			expectedResult: nil,
-		},
-		{
-			name:           "no configuration path",
-			Input:          &Config{},
-			expectedResult: nil,
-		},
 	}
 
-	for idx, test := range tests {
-		err := test.Input.LoadHub()
-		if err == nil && test.err != "" {
-			fmt.Printf("TEST '%s': NOK\n", test.name)
-			t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
-		} else if test.err != "" {
-			if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
-				fmt.Printf("TEST '%s': NOK\n", test.name)
-				t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
-					test.err,
-					fmt.Sprintf("%s", err))
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.input.loadHub()
+			cstest.RequireErrorContains(t, err, tc.expectedErr)
+			if tc.expectedErr != "" {
+				return
 			}
-		}
-		isOk := assert.Equal(t, test.expectedResult, test.Input.Hub)
-		if !isOk {
-			t.Fatalf("TEST '%s': NOK", test.name)
-		} else {
-			fmt.Printf("TEST '%s': OK\n", test.name)
-		}
+
+			assert.Equal(t, tc.expected, tc.input.Hub)
+		})
 	}
 }

pkg/csconfig/profiles.go

@@ -6,10 +6,11 @@ import (
 	"fmt"
 	"io"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/crowdsecurity/go-cs-lib/yamlpatch"
 
 	"github.com/crowdsecurity/crowdsec/pkg/models"
-	"gopkg.in/yaml.v2"
 )
 
 // var OnErrorDefault = OnErrorIgnore
@@ -43,7 +44,6 @@ func (c *LocalApiServerCfg) LoadProfiles() error {
 	}
 	reader := bytes.NewReader(fcontent)
 
-	//process the yaml
 	dec := yaml.NewDecoder(reader)
 	dec.SetStrict(true)
 	for {