Refactor hub management and cscli commands (#2545)

cmd/crowdsec-cli/capi.go

@@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command {
 				return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
 				return err
 			}
 
-			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+			scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 			if err != nil {
 				return fmt.Errorf("failed to get scenarios: %w", err)
 			}

cmd/crowdsec-cli/collections.go

@@ -1,176 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCollectionsCmd() *cobra.Command {
-	var cmdCollections = &cobra.Command{
-		Use:   "collections [action]",
-		Short: "Manage collections from hub",
-		Long:  `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
-		/*TBD fix help*/
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"collection"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	var ignoreError bool
-
-	var cmdCollectionsInstall = &cobra.Command{
-		Use:     "install collection",
-		Short:   "Install given collection(s)",
-		Long:    `Fetch and install given collection(s) from hub`,
-		Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.COLLECTIONS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
-					Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
-	cmdCollections.AddCommand(cmdCollectionsInstall)
-
-	var cmdCollectionsRemove = &cobra.Command{
-		Use:               "remove collection",
-		Short:             "Remove given collection(s)",
-		Long:              `Remove given collection(s) from hub`,
-		Example:           `cscli collections remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one collection to remove or '--all'")
-			}
-
-			for _, name := range args {
-				if !forceAction {
-					item := cwhub.GetItem(cwhub.COLLECTIONS, name)
-					if item == nil {
-						return fmt.Errorf("unable to retrieve: %s", name)
-					}
-					if len(item.BelongsToCollections) > 0 {
-						log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
-						log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
-						continue
-					}
-				}
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
-			}
-			return nil
-		},
-	}
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
-	cmdCollections.AddCommand(cmdCollectionsRemove)
-
-	var cmdCollectionsUpgrade = &cobra.Command{
-		Use:               "upgrade collection",
-		Short:             "Upgrade given collection(s)",
-		Long:              `Fetch and upgrade given collection(s) from hub`,
-		Example:           `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one collection to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
-	cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-	cmdCollections.AddCommand(cmdCollectionsUpgrade)
-
-	var cmdCollectionsInspect = &cobra.Command{
-		Use:               "inspect collection",
-		Short:             "Inspect given collection",
-		Long:              `Inspect given collection`,
-		Example:           `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			for _, name := range args {
-				InspectItem(name, cwhub.COLLECTIONS)
-			}
-		},
-	}
-	cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-	cmdCollections.AddCommand(cmdCollectionsInspect)
-
-	var cmdCollectionsList = &cobra.Command{
-		Use:               "list collection [-a]",
-		Short:             "List all collections",
-		Long:              `List all collections`,
-		Example:           `cscli collections list`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
-		},
-	}
-	cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-	cmdCollections.AddCommand(cmdCollectionsList)
-
-	return cmdCollections
-}

cmd/crowdsec-cli/config_backup.go

@@ -14,21 +14,25 @@ import (
 )
 
 func backupHub(dirPath string) error {
-	var err error
 	var itemDirectory string
 	var upstreamParsers []string
 
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return err
+	}
+
 	for _, itemType := range cwhub.ItemTypes {
 		clog := log.WithFields(log.Fields{
 			"type": itemType,
 		})
-		itemMap := cwhub.GetItemMap(itemType)
+		itemMap := hub.GetItemMap(itemType)
 		if itemMap == nil {
 			clog.Infof("No %s to backup.", itemType)
 			continue
 		}
 		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
-		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+		if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
 			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
 		}
 		upstreamParsers = []string{}
@@ -36,30 +40,30 @@ func backupHub(dirPath string) error {
 			clog = clog.WithFields(log.Fields{
 				"file": v.Name,
 			})
-			if !v.Installed { //only backup installed ones
+			if !v.State.Installed { //only backup installed ones
 				clog.Debugf("[%s] : not installed", k)
 				continue
 			}
 
 			//for the local/tainted ones, we back up the full file
-			if v.Tainted || v.Local || !v.UpToDate {
-				//we need to back up stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+			if v.State.Tainted || v.IsLocal() || !v.State.UpToDate {
+				//we need to backup stages for parsers
+				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
 					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
-					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+					if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
 						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
 					}
 				}
-				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
+				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate)
 				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
-				if err = CopyFile(v.LocalPath, tfile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
+				if err = CopyFile(v.State.LocalPath, tfile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
 				}
-				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
+				clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
 				continue
 			}
-			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
-			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
+			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
+			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
 			upstreamParsers = append(upstreamParsers, v.Name)
 		}
 		//write the upstream items
@@ -100,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
 	parentDir := filepath.Dir(dirPath)
-	if _, err := os.Stat(parentDir); err != nil {
+	if _, err = os.Stat(parentDir); err != nil {
 		return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
 	}
 
@@ -197,10 +201,6 @@ func backupConfigToDirectory(dirPath string) error {
 }
 
 func runConfigBackup(cmd *cobra.Command, args []string) error {
-	if err := require.Hub(csConfig); err != nil {
-		return err
-	}
-
 	if err := backupConfigToDirectory(args[0]); err != nil {
 		return fmt.Errorf("failed to backup config: %w", err)
 	}

cmd/crowdsec-cli/config_restore.go

@@ -21,45 +21,12 @@ type OldAPICfg struct {
 	Password  string `json:"password"`
 }
 
-// it's a rip of the cli version, but in silent-mode
-func silentInstallItem(name string, obtype string) (string, error) {
-	var item = cwhub.GetItem(obtype, name)
-	if item == nil {
-		return "", fmt.Errorf("error retrieving item")
-	}
-	if downloadOnly && item.Downloaded && item.UpToDate {
-		return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
-	}
-	err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
-	if err != nil {
-		return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, *item); err != nil {
-		return "", err
-	}
-
-	if downloadOnly {
-		return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
-	}
-	err = cwhub.EnableItem(csConfig.Hub, item)
-	if err != nil {
-		return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, *item); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("Enabled %s", item.Name), nil
-}
-
 func restoreHub(dirPath string) error {
-	var err error
-
-	if err := csConfig.LoadHub(); err != nil {
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
 		return err
 	}
 
-	cwhub.SetHubBranch()
-
 	for _, itype := range cwhub.ItemTypes {
 		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
 		if _, err = os.Stat(itemDirectory); err != nil {
@@ -78,13 +45,14 @@ func restoreHub(dirPath string) error {
 			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
 		}
 		for _, toinstall := range upstreamList {
-			label, err := silentInstallItem(toinstall, itype)
+			item := hub.GetItem(itype, toinstall)
+			if item == nil {
+				log.Errorf("Item %s/%s not found in hub", itype, toinstall)
+				continue
+			}
+			err := item.Install(false, false)
 			if err != nil {
 				log.Errorf("Error while installing %s : %s", toinstall, err)
-			} else if label != "" {
-				log.Infof("Installed %s : %s", toinstall, label)
-			} else {
-				log.Printf("Installed %s : ok", toinstall)
 			}
 		}
 
@@ -98,7 +66,7 @@ func restoreHub(dirPath string) error {
 			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
 				continue
 			}
-			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
+			if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
 				//we expect a stage here
 				if !file.IsDir() {
 					continue
@@ -302,10 +270,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := require.Hub(csConfig); err != nil {
-		return err
-	}
-
 	if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
 		return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
 	}

cmd/crowdsec-cli/config_show.go

@@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
 cscli:
   - Output                  : {{.Cscli.Output}}
   - Hub Branch              : {{.Cscli.HubBranch}}
-  - Hub Folder              : {{.Cscli.HubDir}}
 {{- end }}
 
 {{- if .API }}

cmd/crowdsec-cli/console.go

@@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba
 				return fmt.Errorf("could not parse CAPI URL: %s", err)
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
 				return err
 			}
 
-			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+			scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 			if err != nil {
 				return fmt.Errorf("failed to get installed scenarios: %s", err)
 			}

cmd/crowdsec-cli/hub.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 
 	"github.com/fatih/color"
@@ -13,30 +12,19 @@ import (
 )
 
 func NewHubCmd() *cobra.Command {
-	var cmdHub = &cobra.Command{
+	cmdHub := &cobra.Command{
 		Use:   "hub [action]",
-		Short: "Manage Hub",
-		Long: `
-Hub management
+		Short: "Manage hub index",
+		Long: `Hub management
 
 List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
-The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
-		`,
-		Example: `
-cscli hub list   # List all installed configurations
-cscli hub update # Download list of available configurations from the hub
-		`,
+The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
+		Example: `cscli hub list
+cscli hub update
+cscli hub upgrade`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			return nil
-		},
 	}
-	cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
 
 	cmdHub.AddCommand(NewHubListCmd())
 	cmdHub.AddCommand(NewHubUpdateCmd())
@@ -45,116 +33,142 @@ cscli hub update # Download list of available configurations from the hub
 	return cmdHub
 }
 
+func runHubList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, v := range hub.Warnings {
+		log.Info(v)
+	}
+
+	for _, line := range hub.ItemStats() {
+		log.Info(line)
+	}
+
+	items := make(map[string][]*cwhub.Item)
+
+	for _, itemType := range cwhub.ItemTypes {
+		items[itemType], err = selectItems(hub, itemType, nil, !all)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = listItems(color.Output, cwhub.ItemTypes, items)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func NewHubListCmd() *cobra.Command {
-	var cmdHubList = &cobra.Command{
+	cmdHubList := &cobra.Command{
 		Use:               "list [-a]",
-		Short:             "List installed configs",
+		Short:             "List all installed configurations",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
+		RunE:              runHubList,
+	}
 
-			// use LocalSync to get warnings about tainted / outdated items
-			warn, _ := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
-			}
-			cwhub.DisplaySummary()
-			ListItems(color.Output, []string{
-				cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
-			}, args, true, false, all)
+	flags := cmdHubList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmdHubList
+}
 
-			return nil
-		},
+func runHubUpdate(cmd *cobra.Command, args []string) error {
+	local := csConfig.Hub
+	remote := require.RemoteHub(csConfig)
+
+	// don't use require.Hub because if there is no index file, it would fail
+	hub, err := cwhub.NewHub(local, remote, true)
+	if err != nil {
+		return fmt.Errorf("failed to update hub: %w", err)
 	}
-	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
 
-	return cmdHubList
+	for _, v := range hub.Warnings {
+		log.Info(v)
+	}
+
+	return nil
 }
 
 func NewHubUpdateCmd() *cobra.Command {
-	var cmdHubUpdate = &cobra.Command{
+	cmdHubUpdate := &cobra.Command{
 		Use:   "update",
-		Short: "Fetch available configs from hub",
+		Short: "Download the latest index (catalog of available configurations)",
 		Long: `
-Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
+Fetches the .index.json file from the hub, containing the list of available configs.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
+		RunE:              runHubUpdate,
+	}
 
-			cwhub.SetHubBranch()
+	return cmdHubUpdate
+}
+
+func runHubUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
+		return err
+	}
 
-			return nil
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
+	for _, itemType := range cwhub.ItemTypes {
+		items, err := hub.GetInstalledItems(itemType)
+		if err != nil {
+			return err
+		}
+
+		updated := 0
+
+		log.Infof("Upgrading %s", itemType)
+		for _, item := range items {
+			didUpdate, err := item.Upgrade(force)
+			if err != nil {
 				return err
 			}
-			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-				if !errors.Is(err, cwhub.ErrIndexNotFound) {
-					return fmt.Errorf("failed to get Hub index : %w", err)
-				}
-				log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
-				cwhub.HubBranch = "master"
-				if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-					return fmt.Errorf("failed to get Hub index after retry: %w", err)
-				}
-			}
-			// use LocalSync to get warnings about tainted / outdated items
-			warn, _ := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
+			if didUpdate {
+				updated++
 			}
-
-			return nil
-		},
+		}
+		log.Infof("Upgraded %d %s", updated, itemType)
 	}
 
-	return cmdHubUpdate
+	return nil
 }
 
 func NewHubUpgradeCmd() *cobra.Command {
-	var cmdHubUpgrade = &cobra.Command{
+	cmdHubUpgrade := &cobra.Command{
 		Use:   "upgrade",
-		Short: "Upgrade all configs installed from hub",
+		Short: "Upgrade all configurations to their latest version",
 		Long: `
 Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			cwhub.SetHubBranch()
-
-			return nil
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			log.Infof("Upgrading collections")
-			cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			log.Infof("Upgrading parsers")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			log.Infof("Upgrading scenarios")
-			cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			log.Infof("Upgrading postoverflows")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-
-			return nil
-		},
+		RunE:              runHubUpgrade,
 	}
-	cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	flags := cmdHubUpgrade.Flags()
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
 
 	return cmdHubUpgrade
 }

cmd/crowdsec-cli/hubtest.go

@@ -18,9 +18,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
-var (
-	HubTest hubtest.HubTest
-)
+var HubTest hubtest.HubTest
 
 func NewHubTestCmd() *cobra.Command {
 	var hubPath string
@@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command {
 			return nil
 		},
 	}
+
 	cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
 	cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
 	cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
@@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command {
 	return cmdHubTest
 }
 
-
 func NewHubTestCreateCmd() *cobra.Command {
 	parsers := []string{}
 	postoverflows := []string{}
@@ -138,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			}
 
 			configFilePath := filepath.Join(testPath, "config.yaml")
-			fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+			fd, err := os.Create(configFilePath)
 			if err != nil {
 				return fmt.Errorf("open: %s", err)
 			}
@@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			return nil
 		},
 	}
+
 	cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
 	cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
 	cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
@@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 	return cmdHubTestCreate
 }
 
-
 func NewHubTestRunCmd() *cobra.Command {
 	var noClean bool
 	var runAll bool
@@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if !runAll && len(args) == 0 {
 				printHelp(cmd)
-				return fmt.Errorf("Please provide test to run or --all flag")
+				return fmt.Errorf("please provide test to run or --all flag")
 			}
 
 			if runAll {
@@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command {
 				}
 			}
 
+			// set timezone to avoid DST issues
+			os.Setenv("TZ", "UTC")
+
 			for _, test := range HubTest.Tests {
 				if csConfig.Cscli.Output == "human" {
 					log.Infof("Running test '%s'", test.Name)
@@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command {
 					}
 				}
 			}
-			if csConfig.Cscli.Output == "human" {
+
+			switch csConfig.Cscli.Output {
+			case "human":
 				hubTestResultTable(color.Output, testResult)
-			} else if csConfig.Cscli.Output == "json" {
+			case "json":
 				jsonResult := make(map[string][]string, 0)
 				jsonResult["success"] = make([]string, 0)
 				jsonResult["fail"] = make([]string, 0)
@@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command {
 					return fmt.Errorf("unable to json test result: %s", err)
 				}
 				fmt.Println(string(jsonStr))
+			default:
+				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			if !success {
@@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command {
 			return nil
 		},
 	}
+
 	cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
 	cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
 	cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
@@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command {
 	return cmdHubTestRun
 }
 
-
 func NewHubTestCleanCmd() *cobra.Command {
 	var cmdHubTestClean = &cobra.Command{
 		Use:               "clean",
@@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command {
 	return cmdHubTestClean
 }
 
-
 func NewHubTestInfoCmd() *cobra.Command {
 	var cmdHubTestInfo = &cobra.Command{
 		Use:               "info",
@@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command {
 	return cmdHubTestInfo
 }
 
-
 func NewHubTestListCmd() *cobra.Command {
 	var cmdHubTestList = &cobra.Command{
 		Use:               "list",
@@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command {
 	return cmdHubTestList
 }
 
-
 func NewHubTestCoverageCmd() *cobra.Command {
 	var showParserCov bool
 	var showScenarioCov bool
@@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				return fmt.Errorf("unable to load all tests: %+v", err)
 			}
 			var err error
-			scenarioCoverage := []hubtest.ScenarioCoverage{}
-			parserCoverage := []hubtest.ParserCoverage{}
+			scenarioCoverage := []hubtest.Coverage{}
+			parserCoverage := []hubtest.Coverage{}
 			scenarioCoveragePercent := 0
 			parserCoveragePercent := 0
 
@@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				parserTested := 0
 				for _, test := range parserCoverage {
 					if test.TestsCount > 0 {
-						parserTested += 1
+						parserTested++
 					}
 				}
 				parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("while getting scenario coverage: %s", err)
 				}
+
 				scenarioTested := 0
 				for _, test := range scenarioCoverage {
 					if test.TestsCount > 0 {
-						scenarioTested += 1
+						scenarioTested++
 					}
 				}
+
 				scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
 			}
 
@@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				os.Exit(0)
 			}
 
-			if csConfig.Cscli.Output == "human" {
+			switch csConfig.Cscli.Output {
+			case "human":
 				if showParserCov || showAll {
 					hubTestParserCoverageTable(color.Output, parserCoverage)
 				}
@@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					fmt.Printf("SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
 				}
-			} else if csConfig.Cscli.Output == "json" {
+			case "json":
 				dump, err := json.MarshalIndent(parserCoverage, "", " ")
 				if err != nil {
 					return err
@@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
 					return err
 				}
 				fmt.Printf("%s", dump)
-			} else {
+			default:
 				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			return nil
 		},
 	}
+
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
 	cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
@@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command {
 	return cmdHubTestCoverage
 }
 
-
 func NewHubTestEvalCmd() *cobra.Command {
 	var evalExpression string
 	var cmdHubTestEval = &cobra.Command{
@@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("can't load test: %+v", err)
 				}
+
 				err = test.ParserAssert.LoadTest(test.ParserResultFile)
 				if err != nil {
 					return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
 				}
+
 				output, err := test.ParserAssert.EvalExpression(evalExpression)
 				if err != nil {
 					return err
 				}
+
 				fmt.Print(output)
 			}
 
 			return nil
 		},
 	}
+
 	cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
 
 	return cmdHubTestEval
 }
 
-
 func NewHubTestExplainCmd() *cobra.Command {
 	var cmdHubTestExplain = &cobra.Command{
 		Use:               "explain",
@@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command {
 				}
 				err = test.ParserAssert.LoadTest(test.ParserResultFile)
 				if err != nil {
-					err := test.Run()
-					if err != nil {
+					if err = test.Run(); err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-					err = test.ParserAssert.LoadTest(test.ParserResultFile)
-					if err != nil {
+
+					if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
 						return fmt.Errorf("unable to load parser result after run: %s", err)
 					}
 				}
 
 				err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
 				if err != nil {
-					err := test.Run()
-					if err != nil {
+					if err = test.Run(); err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-					err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
-					if err != nil {
+
+					if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
 						return fmt.Errorf("unable to load scenario result after run: %s", err)
 					}
 				}

cmd/crowdsec-cli/hubtest_table.go

@@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
 	t.Render()
 }
 
-func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
+func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Parser", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
+
 	for _, test := range coverage {
 		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
 			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-		t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()
 }
 
-func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
+func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Scenario", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
+
 	for _, test := range coverage {
 		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
 			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-		t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()

cmd/crowdsec-cli/item_metrics.go

@@ -0,0 +1,236 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/trace"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func ShowMetrics(hubItem *cwhub.Item) error {
+	switch hubItem.Type {
+	case cwhub.PARSERS:
+		metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		parserMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.SCENARIOS:
+		metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.COLLECTIONS:
+		for _, sub := range hubItem.SubItems() {
+			if err := ShowMetrics(sub); err != nil {
+				return err
+			}
+		}
+	default:
+		// no metrics for this item type
+	}
+	return nil
+}
+
+// GetParserMetric is a complete rip from prom2json
+func GetParserMetric(url string, itemName string) map[string]map[string]int {
+	stats := make(map[string]map[string]int)
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric %v", metric.Labels)
+			} else {
+				if srctype, ok := metric.Labels["type"]; ok {
+					source = srctype + ":" + source
+				}
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_reader_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+					stats[source]["parsed"] = 0
+					stats[source]["reads"] = 0
+					stats[source]["unparsed"] = 0
+					stats[source]["hits"] = 0
+				}
+				stats[source]["reads"] += ival
+			case "cs_parser_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			case "cs_node_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["hits"] += ival
+			case "cs_node_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_node_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetScenarioMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["instantiation"] = 0
+	stats["curr_count"] = 0
+	stats["overflow"] = 0
+	stats["pour"] = 0
+	stats["underflow"] = 0
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_bucket_created_total":
+				stats["instantiation"] += ival
+			case "cs_buckets":
+				stats["curr_count"] += ival
+			case "cs_bucket_overflowed_total":
+				stats["overflow"] += ival
+			case "cs_bucket_poured_total":
+				stats["pour"] += ival
+			case "cs_bucket_underflowed_total":
+				stats["underflow"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetPrometheusMetric(url string) []*prom2json.Family {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+
+	return result
+}
+
+type unit struct {
+	value  int64
+	symbol string
+}
+
+var ranges = []unit{
+	{value: 1e18, symbol: "E"},
+	{value: 1e15, symbol: "P"},
+	{value: 1e12, symbol: "T"},
+	{value: 1e9, symbol: "G"},
+	{value: 1e6, symbol: "M"},
+	{value: 1e3, symbol: "k"},
+	{value: 1, symbol: ""},
+}
+
+func formatNumber(num int) string {
+	goodUnit := unit{}
+	for _, u := range ranges {
+		if int64(num) >= u.value {
+			goodUnit = u
+			break
+		}
+	}
+
+	if goodUnit.value == 1 {
+		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
+	}
+
+	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
+	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
+}

cmd/crowdsec-cli/item_suggest.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/agext/levenshtein"
+	"github.com/spf13/cobra"
+	"slices"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+const MaxDistance = 7
+
+// SuggestNearestMessage returns a message with the most similar item name, if one is found
+func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
+	score := 100
+	nearest := ""
+
+	for _, item := range hub.GetItemMap(itemType) {
+		d := levenshtein.Distance(itemName, item.Name, nil)
+		if d < score {
+			score = d
+			nearest = item.Name
+		}
+	}
+
+	msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
+
+	if score < MaxDistance {
+		msg += fmt.Sprintf(", did you mean '%s'?", nearest)
+	}
+
+	return msg
+}
+
+func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	for _, item := range hub.GetItemMap(itemType) {
+		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
+			comp = append(comp, item.Name)
+		}
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}
+
+func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	items, err := hub.GetInstalledItemNames(itemType)
+	if err != nil {
+		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	if toComplete != "" {
+		for _, item := range items {
+			if strings.Contains(item, toComplete) {
+				comp = append(comp, item)
+			}
+		}
+	} else {
+		comp = items
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}

cmd/crowdsec-cli/itemcommands.go

@@ -0,0 +1,606 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/go-cs-lib/coalesce"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+type cmdHelp struct {
+	// Example is required, the others have a default value
+	// generated from the item type
+	use     string
+	short   string
+	long    string
+	example string
+}
+
+type hubItemType struct {
+	name        string // plural, as used in the hub index
+	singular    string
+	oneOrMore   string // parenthetical pluralizaion: "parser(s)"
+	help        cmdHelp
+	installHelp cmdHelp
+	removeHelp  cmdHelp
+	upgradeHelp cmdHelp
+	inspectHelp cmdHelp
+	listHelp    cmdHelp
+}
+
+var hubItemTypes = map[string]hubItemType{
+	"parsers": {
+		name:      "parsers",
+		singular:  "parser",
+		oneOrMore: "parser(s)",
+		help: cmdHelp{
+			example: `cscli parsers list -a
+cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli parsers list
+cscli parsers list -a
+cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+
+List only enabled parsers unless "-a" or names are specified.`,
+		},
+	},
+	"postoverflows": {
+		name:      "postoverflows",
+		singular:  "postoverflow",
+		oneOrMore: "postoverflow(s)",
+		help: cmdHelp{
+			example: `cscli postoverflows list -a
+cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli postoverflows list
+cscli postoverflows list -a
+cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
+
+List only enabled postoverflows unless "-a" or names are specified.`,
+		},
+	},
+	"scenarios": {
+		name:      "scenarios",
+		singular:  "scenario",
+		oneOrMore: "scenario(s)",
+		help: cmdHelp{
+			example: `cscli scenarios list -a
+cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli scenarios list
+cscli scenarios list -a
+cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
+
+List only enabled scenarios unless "-a" or names are specified.`,
+		},
+	},
+	"collections": {
+		name:      "collections",
+		singular:  "collection",
+		oneOrMore: "collection(s)",
+		help: cmdHelp{
+			example: `cscli collections list -a
+cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli collections list
+cscli collections list -a
+cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
+
+List only enabled collections unless "-a" or names are specified.`,
+		},
+	},
+}
+
+func NewItemsCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
+		Short:             coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
+		Long:              it.help.long,
+		Example:           it.help.example,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{it.singular},
+		DisableAutoGenTag: true,
+	}
+
+	cmd.AddCommand(NewItemsInstallCmd(typeName))
+	cmd.AddCommand(NewItemsRemoveCmd(typeName))
+	cmd.AddCommand(NewItemsUpgradeCmd(typeName))
+	cmd.AddCommand(NewItemsInspectCmd(typeName))
+	cmd.AddCommand(NewItemsListCmd(typeName))
+
+	return cmd
+}
+
+func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		downloadOnly, err := flags.GetBool("download-only")
+		if err != nil {
+			return err
+		}
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		ignoreError, err := flags.GetBool("ignore")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+		if err != nil {
+			return err
+		}
+
+		for _, name := range args {
+			item := hub.GetItem(it.name, name)
+			if item == nil {
+				msg := SuggestNearestMessage(hub, it.name, name)
+				if !ignoreError {
+					return fmt.Errorf(msg)
+				}
+
+				log.Errorf(msg)
+
+				continue
+			}
+
+			if err := item.Install(force, downloadOnly); err != nil {
+				if !ignoreError {
+					return fmt.Errorf("error while installing '%s': %w", item.Name, err)
+				}
+				log.Errorf("Error while installing '%s': %s", item.Name, err)
+			}
+		}
+
+		log.Infof(ReloadMessage())
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsInstallCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.installHelp.use, "install [item]..."),
+		Short:             coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
+		Example:           it.installHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(typeName, args, toComplete)
+		},
+		RunE: itemsInstallRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
+
+	return cmd
+}
+
+// return the names of the installed parents of an item, used to check if we can remove it
+func istalledParentNames(item *cwhub.Item) []string {
+	ret := make([]string, 0)
+
+	for _, parent := range item.Ancestors() {
+		if parent.State.Installed {
+			ret = append(ret, parent.Name)
+		}
+	}
+
+	return ret
+}
+
+func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		purge, err := flags.GetBool("purge")
+		if err != nil {
+			return err
+		}
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, nil)
+		if err != nil {
+			return err
+		}
+
+		if all {
+			getter := hub.GetInstalledItems
+			if purge {
+				getter = hub.GetAllItems
+			}
+
+			items, err := getter(it.name)
+			if err != nil {
+				return err
+			}
+
+			removed := 0
+
+			for _, item := range items {
+				didRemove, err := item.Remove(purge, force)
+				if err != nil {
+					return err
+				}
+				if didRemove {
+					removed++
+				}
+			}
+
+			log.Infof("Removed %d %s", removed, it.name)
+			if removed > 0 {
+				log.Infof(ReloadMessage())
+			}
+
+			return nil
+		}
+
+		if len(args) == 0 {
+			return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
+		}
+
+		removed := 0
+
+		for _, itemName := range args {
+			item := hub.GetItem(it.name, itemName)
+			if item == nil {
+				return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
+			}
+
+			parents := istalledParentNames(item)
+
+			if !force && len(parents) > 0 {
+				log.Warningf("%s belongs to collections: %s", item.Name, parents)
+				log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
+				continue
+			}
+
+			didRemove, err := item.Remove(purge, force)
+			if err != nil {
+				return err
+			}
+
+			if didRemove {
+				log.Infof("Removed %s", item.Name)
+				removed++
+			}
+		}
+		if removed > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+	return run
+}
+
+func NewItemsRemoveCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.removeHelp.use, "remove [item]..."),
+		Short:             coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
+		Example:           it.removeHelp.example,
+		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsRemoveRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
+
+	return cmd
+}
+
+func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+		if err != nil {
+			return err
+		}
+
+		if all {
+			items, err := hub.GetInstalledItems(it.name)
+			if err != nil {
+				return err
+			}
+
+			updated := 0
+
+			for _, item := range items {
+				didUpdate, err := item.Upgrade(force)
+				if err != nil {
+					return err
+				}
+				if didUpdate {
+					updated++
+				}
+			}
+
+			log.Infof("Updated %d %s", updated, it.name)
+
+			if updated > 0 {
+				log.Infof(ReloadMessage())
+			}
+
+			return nil
+		}
+
+		if len(args) == 0 {
+			return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
+		}
+
+		updated := 0
+
+		for _, itemName := range args {
+			item := hub.GetItem(it.name, itemName)
+			if item == nil {
+				return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
+			}
+
+			didUpdate, err := item.Upgrade(force)
+			if err != nil {
+				return err
+			}
+
+			if didUpdate {
+				log.Infof("Updated %s", item.Name)
+				updated++
+			}
+		}
+		if updated > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsUpgradeCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
+		Short:             coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
+		Example:           it.upgradeHelp.example,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsUpgradeRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmd
+}
+
+func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		url, err := flags.GetString("url")
+		if err != nil {
+			return err
+		}
+
+		if url != "" {
+			csConfig.Cscli.PrometheusUrl = url
+		}
+
+		noMetrics, err := flags.GetBool("no-metrics")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, nil)
+		if err != nil {
+			return err
+		}
+
+		for _, name := range args {
+			item := hub.GetItem(it.name, name)
+			if item == nil {
+				return fmt.Errorf("can't find '%s' in %s", name, it.name)
+			}
+			if err = InspectItem(item, !noMetrics); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsInspectCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.inspectHelp.use, "inspect [item]..."),
+		Short:             coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
+		Example:           it.inspectHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsInspectRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
+
+	return cmd
+}
+
+func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := require.Hub(csConfig, nil)
+		if err != nil {
+			return err
+		}
+
+		items := make(map[string][]*cwhub.Item)
+
+		items[it.name], err = selectItems(hub, it.name, args, !all)
+		if err != nil {
+			return err
+		}
+
+		if err = listItems(color.Output, []string{it.name}, items); err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsListCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.listHelp.use, "list [item... | -a]"),
+		Short:             coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
+		Long:              coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
+		Example:           it.listHelp.example,
+		DisableAutoGenTag: true,
+		RunE:              itemsListRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmd
+}

cmd/crowdsec-cli/items.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+	"slices"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
+func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
+	itemNames := hub.GetItemNames(itemType)
+
+	notExist := []string{}
+
+	if len(args) > 0 {
+		for _, arg := range args {
+			if !slices.Contains(itemNames, arg) {
+				notExist = append(notExist, arg)
+			}
+		}
+	}
+
+	if len(notExist) > 0 {
+		return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
+	}
+
+	if len(args) > 0 {
+		itemNames = args
+		installedOnly = false
+	}
+
+	items := make([]*cwhub.Item, 0, len(itemNames))
+
+	for _, itemName := range itemNames {
+		item := hub.GetItem(itemType, itemName)
+		if installedOnly && !item.State.Installed {
+			continue
+		}
+
+		items = append(items, item)
+	}
+
+	cwhub.SortItemSlice(items)
+
+	return items, nil
+}
+
+func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
+	switch csConfig.Cscli.Output {
+	case "human":
+		for _, itemType := range itemTypes {
+			listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
+		}
+	case "json":
+		type itemHubStatus struct {
+			Name         string `json:"name"`
+			LocalVersion string `json:"local_version"`
+			LocalPath    string `json:"local_path"`
+			Description  string `json:"description"`
+			UTF8Status   string `json:"utf8_status"`
+			Status       string `json:"status"`
+		}
+
+		hubStatus := make(map[string][]itemHubStatus)
+		for _, itemType := range itemTypes {
+			// empty slice in case there are no items of this type
+			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
+
+			for i, item := range items[itemType] {
+				status, emo := item.InstallStatus()
+				hubStatus[itemType][i] = itemHubStatus{
+					Name:         item.Name,
+					LocalVersion: item.State.LocalVersion,
+					LocalPath:    item.State.LocalPath,
+					Description:  item.Description,
+					Status:       status,
+					UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
+				}
+			}
+		}
+
+		x, err := json.MarshalIndent(hubStatus, "", " ")
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal: %w", err)
+		}
+
+		out.Write(x)
+	case "raw":
+		csvwriter := csv.NewWriter(out)
+
+		header := []string{"name", "status", "version", "description"}
+		if len(itemTypes) > 1 {
+			header = append(header, "type")
+		}
+
+		if err := csvwriter.Write(header); err != nil {
+			return fmt.Errorf("failed to write header: %s", err)
+		}
+
+		for _, itemType := range itemTypes {
+			for _, item := range items[itemType] {
+				status, _ := item.InstallStatus()
+				row := []string{
+					item.Name,
+					status,
+					item.State.LocalVersion,
+					item.Description,
+				}
+				if len(itemTypes) > 1 {
+					row = append(row, itemType)
+				}
+				if err := csvwriter.Write(row); err != nil {
+					return fmt.Errorf("failed to write raw output: %s", err)
+				}
+			}
+		}
+		csvwriter.Flush()
+	default:
+		return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
+	}
+
+	return nil
+}
+
+func InspectItem(item *cwhub.Item, showMetrics bool) error {
+	switch csConfig.Cscli.Output {
+	case "human", "raw":
+		enc := yaml.NewEncoder(os.Stdout)
+		enc.SetIndent(2)
+		if err := enc.Encode(item); err != nil {
+			return fmt.Errorf("unable to encode item: %s", err)
+		}
+	case "json":
+		b, err := json.MarshalIndent(*item, "", "  ")
+		if err != nil {
+			return fmt.Errorf("unable to marshal item: %s", err)
+		}
+		fmt.Print(string(b))
+	}
+
+	if csConfig.Cscli.Output == "human" && showMetrics {
+		fmt.Printf("\nCurrent metrics: \n")
+		if err := ShowMetrics(item); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/crowdsec-cli/lapi.go

@@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
 	}
 
-	if err := require.Hub(csConfig); err != nil {
+	hub, err := require.Hub(csConfig, nil)
+	if err != nil {
 		log.Fatal(err)
 	}
 
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		log.Fatalf("failed to get scenarios : %s", err)
 	}
@@ -338,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs
 				log.Fatalf("Failed to init expr helpers : %s", err)
 			}
 
-			// Populate cwhub package tools
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Fatalf("Failed to load hub index : %s", err)
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
+				log.Fatal(err)
 			}
 
-			csParsers := parser.NewParsers()
+			csParsers := parser.NewParsers(hub)
 			if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
 				log.Fatalf("unable to load parsers: %s", err)
 			}

cmd/crowdsec-cli/main.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"slices"
 	"strings"
 
 	"github.com/fatih/color"
@@ -12,9 +11,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
+	"slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
@@ -29,15 +28,11 @@ var dbClient *database.Client
 var OutputFormat string
 var OutputColor string
 
-var downloadOnly bool
-var forceAction bool
-var purge bool
-var all bool
-
-var prometheusURL string
-
 var mergedConfig string
 
+// flagBranch overrides the value in csConfig.Cscli.HubBranch
+var flagBranch = ""
+
 func initConfig() {
 	var err error
 	if trace_lvl {
@@ -58,9 +53,6 @@ func initConfig() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		if err := csConfig.LoadCSCLI(); err != nil {
-			log.Fatal(err)
-		}
 	} else {
 		csConfig = csconfig.NewDefaultConfig()
 	}
@@ -71,13 +63,10 @@ func initConfig() {
 		log.Debugf("Enabled feature flags: %s", fflist)
 	}
 
-	if csConfig.Cscli == nil {
-		log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
+	if flagBranch != "" {
+		csConfig.Cscli.HubBranch = flagBranch
 	}
 
-	if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
-		cwhub.HubBranch = csConfig.Cscli.HubBranch
-	}
 	if OutputFormat != "" {
 		csConfig.Cscli.Output = OutputFormat
 		if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
@@ -206,7 +195,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
 	rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
 
-	rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
+	rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
 	if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
 		log.Fatalf("failed to hide flag: %s", err)
 	}
@@ -243,10 +232,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewSimulationCmds())
 	rootCmd.AddCommand(NewBouncersCmd())
 	rootCmd.AddCommand(NewMachinesCmd())
-	rootCmd.AddCommand(NewParsersCmd())
-	rootCmd.AddCommand(NewScenariosCmd())
-	rootCmd.AddCommand(NewCollectionsCmd())
-	rootCmd.AddCommand(NewPostOverflowsCmd())
 	rootCmd.AddCommand(NewCapiCmd())
 	rootCmd.AddCommand(NewLapiCmd())
 	rootCmd.AddCommand(NewCompletionCmd())
@@ -255,6 +240,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewHubTestCmd())
 	rootCmd.AddCommand(NewNotificationsCmd())
 	rootCmd.AddCommand(NewSupportCmd())
+	rootCmd.AddCommand(NewItemsCmd("collections"))
+	rootCmd.AddCommand(NewItemsCmd("parsers"))
+	rootCmd.AddCommand(NewItemsCmd("scenarios"))
+	rootCmd.AddCommand(NewItemsCmd("postoverflows"))
 
 	if fflag.CscliSetup.IsEnabled() {
 		rootCmd.AddCommand(NewSetupCmd())

cmd/crowdsec-cli/metrics.go

@@ -284,29 +284,32 @@ var noUnit bool
 
 
 func runMetrics(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadPrometheus(); err != nil {
-		return fmt.Errorf("failed to load prometheus config: %w", err)
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
 	}
 
-	if csConfig.Prometheus == nil {
-		return fmt.Errorf("prometheus section missing, can't show metrics")
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
 	}
 
-	if !csConfig.Prometheus.Enabled {
-		return fmt.Errorf("prometheus is not enabled, can't show metrics")
+	noUnit, err = flags.GetBool("no-unit")
+	if err != nil {
+		return err
 	}
 
-	if prometheusURL == "" {
-		prometheusURL = csConfig.Cscli.PrometheusUrl
+	if csConfig.Prometheus == nil {
+		return fmt.Errorf("prometheus section missing, can't show metrics")
 	}
 
-	if prometheusURL == "" {
-		return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
+	if !csConfig.Prometheus.Enabled {
+		return fmt.Errorf("prometheus is not enabled, can't show metrics")
 	}
 
-	err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
-	if err != nil {
-		return fmt.Errorf("could not fetch prometheus metrics: %w", err)
+	if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
+		return err
 	}
 	return nil
 }
@@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: runMetrics,
 	}
-	cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
-	cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
+
+	flags := cmdMetrics.PersistentFlags()
+	flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
+	flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
 
 	return cmdMetrics
 }

cmd/crowdsec-cli/parsers.go

@@ -1,194 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewParsersCmd() *cobra.Command {
-	var cmdParsers = &cobra.Command{
-		Use:   "parsers [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
-		Example: `cscli parsers install crowdsecurity/sshd-logs
-cscli parsers inspect crowdsecurity/sshd-logs
-cscli parsers upgrade crowdsecurity/sshd-logs
-cscli parsers list
-cscli parsers remove crowdsecurity/sshd-logs
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"parser"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdParsers.AddCommand(NewParsersInstallCmd())
-	cmdParsers.AddCommand(NewParsersRemoveCmd())
-	cmdParsers.AddCommand(NewParsersUpgradeCmd())
-	cmdParsers.AddCommand(NewParsersInspectCmd())
-	cmdParsers.AddCommand(NewParsersListCmd())
-
-	return cmdParsers
-}
-
-func NewParsersInstallCmd() *cobra.Command {
-	var ignoreError bool
-
-	var cmdParsersInstall = &cobra.Command{
-		Use:               "install [config]",
-		Short:             "Install given parser(s)",
-		Long:              `Fetch and install given parser(s) from hub`,
-		Example:           `cscli parsers install crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS, name)
-					Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
-
-	return cmdParsersInstall
-}
-
-func NewParsersRemoveCmd() *cobra.Command {
-	cmdParsersRemove := &cobra.Command{
-		Use:               "remove [config]",
-		Short:             "Remove given parser(s)",
-		Long:              `Remove given parse(s) from hub`,
-		Example:           `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one parser to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
-			}
-
-			return nil
-		},
-	}
-
-	cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
-
-	return cmdParsersRemove
-}
-
-func NewParsersUpgradeCmd() *cobra.Command {
-	cmdParsersUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given parser(s)",
-		Long:              `Fetch and upgrade given parser(s) from hub`,
-		Example:           `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one parser to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdParsersUpgrade
-}
-
-func NewParsersInspectCmd() *cobra.Command {
-	var cmdParsersInspect = &cobra.Command{
-		Use:               "inspect [name]",
-		Short:             "Inspect given parser",
-		Long:              `Inspect given parser`,
-		Example:           `cscli parsers inspect crowdsec/xxx`,
-		DisableAutoGenTag: true,
-		Args:              cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS)
-		},
-	}
-
-	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-
-	return cmdParsersInspect
-}
-
-func NewParsersListCmd() *cobra.Command {
-	var cmdParsersList = &cobra.Command{
-		Use:   "list [name]",
-		Short: "List all parsers or given one",
-		Long:  `List all parsers or given one`,
-		Example: `cscli parsers list
-cscli parser list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
-		},
-	}
-
-	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdParsersList
-}

cmd/crowdsec-cli/postoverflows.go

@@ -1,191 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewPostOverflowsCmd() *cobra.Command {
-	cmdPostOverflows := &cobra.Command{
-		Use:   "postoverflows [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
-		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
-		cscli postoverflows inspect crowdsecurity/cdn-whitelist
-		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
-		cscli postoverflows list
-		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"postoverflow"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
-
-	return cmdPostOverflows
-}
-
-func NewPostOverflowsInstallCmd() *cobra.Command {
-	var ignoreError bool
-
-	cmdPostOverflowsInstall := &cobra.Command{
-		Use:               "install [config]",
-		Short:             "Install given postoverflow(s)",
-		Long:              `Fetch and install given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
-					Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
-
-	return cmdPostOverflowsInstall
-}
-
-func NewPostOverflowsRemoveCmd() *cobra.Command {
-	cmdPostOverflowsRemove := &cobra.Command{
-		Use:               "remove [config]",
-		Short:             "Remove given postoverflow(s)",
-		Long:              `remove given postoverflow(s)`,
-		Example:           `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
-			}
-
-			return nil
-		},
-	}
-
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
-
-	return cmdPostOverflowsRemove
-}
-
-func NewPostOverflowsUpgradeCmd() *cobra.Command {
-	cmdPostOverflowsUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given postoverflow(s)",
-		Long:              `Fetch and Upgrade given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdPostOverflowsUpgrade
-}
-
-func NewPostOverflowsInspectCmd() *cobra.Command {
-	cmdPostOverflowsInspect := &cobra.Command{
-		Use:               "inspect [config]",
-		Short:             "Inspect given postoverflow",
-		Long:              `Inspect given postoverflow`,
-		Example:           `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		Args:              cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS_OVFLW)
-		},
-	}
-
-	return cmdPostOverflowsInspect
-}
-
-func NewPostOverflowsListCmd() *cobra.Command {
-	cmdPostOverflowsList := &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all postoverflows or given one",
-		Long:  `List all postoverflows or given one`,
-		Example: `cscli postoverflows list
-cscli postoverflows list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
-		},
-	}
-
-	cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdPostOverflowsList
-}

cmd/crowdsec-cli/require/branch.go

@@ -0,0 +1,58 @@
+package require
+
+// Set the appropriate hub branch according to config settings and crowdsec version
+
+import (
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+)
+
+func chooseBranch(cfg *csconfig.Config) string {
+	// this was set from config.yaml or flag
+	if cfg.Cscli.HubBranch != "" {
+		log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
+		return cfg.Cscli.HubBranch
+	}
+
+	latest, err := cwversion.Latest()
+	if err != nil {
+		log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
+		return "master"
+	}
+
+	csVersion := cwversion.VersionStrip()
+	if csVersion == latest {
+		log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
+		return "master"
+	}
+
+	// if current version is greater than the latest we are in pre-release
+	if semver.Compare(csVersion, latest) == 1 {
+		log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
+		return "master"
+	}
+
+	if csVersion == "" {
+		log.Warning("Crowdsec version is not set, using hub branch 'master'")
+		return "master"
+	}
+
+	log.Warnf("A new CrowdSec release is available (%s). "+
+		"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
+		latest, csVersion)
+	return csVersion
+}
+
+
+// HubBranch sets the branch (in cscli config) and returns its value
+// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
+func HubBranch(cfg *csconfig.Config) string {
+	branch := chooseBranch(cfg)
+
+	cfg.Cscli.HubBranch = branch
+
+	return branch
+}

cmd/crowdsec-cli/require/require.go

@@ -23,6 +23,7 @@ func CAPI(c *csconfig.Config) error {
 	if c.API.Server.OnlineClient == nil {
 		return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
 	}
+
 	return nil
 }
 
@@ -30,6 +31,7 @@ func PAPI(c *csconfig.Config) error {
 	if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
 		return fmt.Errorf("no PAPI URL in configuration")
 	}
+
 	return nil
 }
 
@@ -45,6 +47,7 @@ func DB(c *csconfig.Config) error {
 	if err := c.LoadDBConfig(); err != nil {
 		return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
 	}
+
 	return nil
 }
 
@@ -64,20 +67,33 @@ func Notifications(c *csconfig.Config) error {
 	return nil
 }
 
-func Hub (c *csconfig.Config) error {
-	if err := c.LoadHub(); err != nil {
-		return err
+// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
+func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
+	// set branch in config, and log if necessary
+	branch := HubBranch(c)
+	remote := &cwhub.RemoteHubCfg {
+		Branch: branch,
+		URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
+		// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
+		IndexPath: ".index.json",
 	}
 
-	if c.Hub == nil {
-		return fmt.Errorf("you must configure cli before interacting with hub")
-	}
+	return remote
+}
 
-	cwhub.SetHubBranch()
+// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
+// If no remote parameter is provided, the hub can only be used for local operations.
+func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
+	local := c.Hub
 
-	if err := cwhub.GetHubIdx(c.Hub); err != nil {
-		return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
+	if local == nil {
+		return nil, fmt.Errorf("you must configure cli before interacting with hub")
 	}
 
-	return nil
+	hub, err := cwhub.NewHub(local, remote, false)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
+	}
+
+	return hub, nil
 }

cmd/crowdsec-cli/scenarios.go

@@ -1,188 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewScenariosCmd() *cobra.Command {
-	var cmdScenarios = &cobra.Command{
-		Use:   "scenarios [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
-		Example: `cscli scenarios list [-a]
-cscli scenarios install crowdsecurity/ssh-bf
-cscli scenarios inspect crowdsecurity/ssh-bf
-cscli scenarios upgrade crowdsecurity/ssh-bf
-cscli scenarios remove crowdsecurity/ssh-bf
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"scenario"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdScenarios.AddCommand(NewCmdScenariosInstall())
-	cmdScenarios.AddCommand(NewCmdScenariosRemove())
-	cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
-	cmdScenarios.AddCommand(NewCmdScenariosInspect())
-	cmdScenarios.AddCommand(NewCmdScenariosList())
-
-	return cmdScenarios
-}
-
-func NewCmdScenariosInstall() *cobra.Command {
-	var ignoreError bool
-
-	var cmdScenariosInstall = &cobra.Command{
-		Use:     "install [config]",
-		Short:   "Install given scenario(s)",
-		Long:    `Fetch and install given scenario(s) from hub`,
-		Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.SCENARIOS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
-					Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
-
-	return cmdScenariosInstall
-}
-
-func NewCmdScenariosRemove() *cobra.Command {
-	var cmdScenariosRemove = &cobra.Command{
-		Use:     "remove [config]",
-		Short:   "Remove given scenario(s)",
-		Long:    `remove given scenario(s)`,
-		Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
-		Aliases: []string{"delete"},
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one scenario to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
-			}
-			return nil
-		},
-	}
-	cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
-
-	return cmdScenariosRemove
-}
-
-func NewCmdScenariosUpgrade() *cobra.Command {
-	var cmdScenariosUpgrade = &cobra.Command{
-		Use:     "upgrade [config]",
-		Short:   "Upgrade given scenario(s)",
-		Long:    `Fetch and Upgrade given scenario(s) from hub`,
-		Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
-	cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdScenariosUpgrade
-}
-
-func NewCmdScenariosInspect() *cobra.Command {
-	var cmdScenariosInspect = &cobra.Command{
-		Use:     "inspect [config]",
-		Short:   "Inspect given scenario",
-		Long:    `Inspect given scenario`,
-		Example: `cscli scenarios inspect crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.SCENARIOS)
-		},
-	}
-	cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-
-	return cmdScenariosInspect
-}
-
-func NewCmdScenariosList() *cobra.Command {
-	var cmdScenariosList = &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all scenario(s) or given one",
-		Long:  `List all scenario(s) or given one`,
-		Example: `cscli scenarios list
-cscli scenarios list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
-		},
-	}
-	cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdScenariosList
-}

cmd/crowdsec-cli/setup.go

@@ -6,13 +6,15 @@ import (
 	"os"
 	"os/exec"
 
+	goccyyaml "github.com/goccy/go-yaml"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
-	goccyyaml "github.com/goccy/go-yaml"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/setup"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 // NewSetupCmd defines the "cscli setup" command.
@@ -303,7 +305,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("while reading file %s: %w", fromFile, err)
 	}
 
-	if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
+	if err != nil {
+		return err
+	}
+
+	if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/simulation.go

@@ -3,11 +3,11 @@ package main
 import (
 	"fmt"
 	"os"
-	"slices"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
+	"slices"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@@ -112,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
 			if err := csConfig.LoadSimulation(); err != nil {
 				log.Fatal(err)
 			}
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before using simulation")
-			}
 			if csConfig.Cscli.SimulationConfig == nil {
 				return fmt.Errorf("no simulation configured")
 			}
@@ -145,18 +142,19 @@ func NewSimulationEnableCmd() *cobra.Command {
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
 				log.Fatal(err)
 			}
 
 			if len(args) > 0 {
 				for _, scenario := range args {
-					var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
+					var item = hub.GetItem(cwhub.SCENARIOS, scenario)
 					if item == nil {
 						log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
 						continue
 					}
-					if !item.Installed {
+					if !item.State.Installed {
 						log.Warningf("'%s' isn't enabled", scenario)
 					}
 					isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)

cmd/crowdsec-cli/support.go

@@ -58,10 +58,6 @@ func stripAnsiString(str string) string {
 
 func collectMetrics() ([]byte, []byte, error) {
 	log.Info("Collecting prometheus metrics")
-	err := csConfig.LoadPrometheus()
-	if err != nil {
-		return nil, nil, err
-	}
 
 	if csConfig.Cscli.PrometheusUrl == "" {
 		log.Warn("No Prometheus URL configured, metrics will not be collected")
@@ -69,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) {
 	}
 
 	humanMetrics := bytes.NewBuffer(nil)
-	err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
+	err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
 
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
 	}
 
-	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
+	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
 	}
@@ -132,10 +128,21 @@ func collectOSInfo() ([]byte, error) {
 	return w.Bytes(), nil
 }
 
-func collectHubItems(itemType string) []byte {
+func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
+	var err error
+
 	out := bytes.NewBuffer(nil)
 	log.Infof("Collecting %s list", itemType)
-	ListItems(out, []string{itemType}, []string{}, false, true, all)
+
+	items := make(map[string][]*cwhub.Item)
+
+	if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
+
+	if err := listItems(out, []string{itemType}, items); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
 	return out.Bytes()
 }
 
@@ -157,7 +164,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
 	return out.Bytes(), nil
 }
 
-func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
+func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
 	if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
 		return []byte("No agent credentials found, are we LAPI ?")
 	}
@@ -167,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 	if err != nil {
 		return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
 	}
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
 	}
@@ -295,7 +302,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				skipAgent = true
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil)
+			if err != nil {
 				log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 				skipHub = true
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
@@ -333,10 +341,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
 			infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
 
 			if !skipHub {
-				infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
-				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
-				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
-				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
+				infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
+				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
+				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
+				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
 			}
 
 			if !skipDB {
@@ -358,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
 					csConfig.API.Server.OnlineClient.Credentials.Password,
 					csConfig.API.Server.OnlineClient.Credentials.URL,
-					CAPIURLPrefix)
+					CAPIURLPrefix,
+					hub)
 			}
 
 			if !skipLAPI {
@@ -366,7 +375,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
 					csConfig.API.Client.Credentials.Password,
 					csConfig.API.Client.Credentials.URL,
-					LAPIURLPrefix)
+					LAPIURLPrefix,
+					hub)
 				infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
 			}
 

cmd/crowdsec-cli/utils.go

@@ -1,36 +1,17 @@
 package main
 
 import (
-	"encoding/csv"
-	"encoding/json"
 	"fmt"
-	"io"
-	"math"
 	"net"
-	"net/http"
-	"slices"
-	"strconv"
 	"strings"
-	"time"
 
-	"github.com/fatih/color"
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/agext/levenshtein"
-	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-const MaxDistance = 7
-
 func printHelp(cmd *cobra.Command) {
 	err := cmd.Help()
 	if err != nil {
@@ -38,197 +19,6 @@ func printHelp(cmd *cobra.Command) {
 	}
 }
 
-func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
-	errMsg := ""
-	if score < MaxDistance {
-		errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
-	} else {
-		errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
-	}
-	if ignoreErr {
-		log.Error(errMsg)
-	} else {
-		log.Fatalf(errMsg)
-	}
-}
-
-func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
-	allItems := make([]string, 0)
-	nearestScore := 100
-	nearestItem := &cwhub.Item{}
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		allItems = append(allItems, item.Name)
-	}
-
-	for _, s := range allItems {
-		d := levenshtein.Distance(itemName, s, nil)
-		if d < nearestScore {
-			nearestScore = d
-			nearestItem = cwhub.GetItem(itemType, s)
-		}
-	}
-	return nearestItem, nearestScore
-}
-
-func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
-			comp = append(comp, item.Name)
-		}
-	}
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	items, err := cwhub.GetInstalledItemsAsString(itemType)
-	if err != nil {
-		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-
-	if toComplete != "" {
-		for _, item := range items {
-			if strings.Contains(item, toComplete) {
-				comp = append(comp, item)
-			}
-		}
-	} else {
-		comp = items
-	}
-
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
-	var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
-
-	for _, itemType := range itemTypes {
-		itemName := ""
-		if len(args) == 1 {
-			itemName = args[0]
-		}
-		hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
-	}
-
-	if csConfig.Cscli.Output == "human" {
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
-		}
-	} else if csConfig.Cscli.Output == "json" {
-		x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
-		if err != nil {
-			log.Fatalf("failed to unmarshal")
-		}
-		out.Write(x)
-	} else if csConfig.Cscli.Output == "raw" {
-		csvwriter := csv.NewWriter(out)
-		if showHeader {
-			header := []string{"name", "status", "version", "description"}
-			if showType {
-				header = append(header, "type")
-			}
-			err := csvwriter.Write(header)
-			if err != nil {
-				log.Fatalf("failed to write header: %s", err)
-			}
-
-		}
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			for _, status := range statuses {
-				if status.LocalVersion == "" {
-					status.LocalVersion = "n/a"
-				}
-				row := []string{
-					status.Name,
-					status.Status,
-					status.LocalVersion,
-					status.Description,
-				}
-				if showType {
-					row = append(row, itemType)
-				}
-				err := csvwriter.Write(row)
-				if err != nil {
-					log.Fatalf("failed to write raw output : %s", err)
-				}
-			}
-		}
-		csvwriter.Flush()
-	}
-}
-
-func InspectItem(name string, objecitemType string) {
-
-	hubItem := cwhub.GetItem(objecitemType, name)
-	if hubItem == nil {
-		log.Fatalf("unable to retrieve item.")
-	}
-	var b []byte
-	var err error
-	switch csConfig.Cscli.Output {
-	case "human", "raw":
-		b, err = yaml.Marshal(*hubItem)
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	case "json":
-		b, err = json.MarshalIndent(*hubItem, "", " ")
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	}
-	fmt.Printf("%s", string(b))
-	if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
-		return
-	}
-
-	if prometheusURL == "" {
-		//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
-		//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
-		listenAddr := csConfig.Prometheus.ListenAddr
-		if listenAddr == "" {
-			listenAddr = "127.0.0.1"
-		}
-		listenPort := csConfig.Prometheus.ListenPort
-		if listenPort == 0 {
-			listenPort = 6060
-		}
-		prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
-		log.Debugf("No prometheus URL provided using: %s", prometheusURL)
-	}
-
-	fmt.Printf("\nCurrent metrics : \n")
-	ShowMetrics(hubItem)
-}
-
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
 
 	/*if a range is provided, change the scope*/
@@ -259,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 	return nil
 }
 
-func ShowMetrics(hubItem *cwhub.Item) {
-	switch hubItem.Type {
-	case cwhub.PARSERS:
-		metrics := GetParserMetric(prometheusURL, hubItem.Name)
-		parserMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.SCENARIOS:
-		metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
-		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.COLLECTIONS:
-		for _, item := range hubItem.Parsers {
-			metrics := GetParserMetric(prometheusURL, item)
-			parserMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Scenarios {
-			metrics := GetScenarioMetric(prometheusURL, item)
-			scenarioMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Collections {
-			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
-			if hubItem == nil {
-				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
-			}
-			ShowMetrics(hubItem)
-		}
-	default:
-		log.Errorf("item of type '%s' is unknown", hubItem.Type)
-	}
-}
-
-// GetParserMetric is a complete rip from prom2json
-func GetParserMetric(url string, itemName string) map[string]map[string]int {
-	stats := make(map[string]map[string]int)
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			source, ok := metric.Labels["source"]
-			if !ok {
-				log.Debugf("no source in Metric %v", metric.Labels)
-			} else {
-				if srctype, ok := metric.Labels["type"]; ok {
-					source = srctype + ":" + source
-				}
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_reader_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-					stats[source]["parsed"] = 0
-					stats[source]["reads"] = 0
-					stats[source]["unparsed"] = 0
-					stats[source]["hits"] = 0
-				}
-				stats[source]["reads"] += ival
-			case "cs_parser_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_parser_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			case "cs_node_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["hits"] += ival
-			case "cs_node_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_node_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetScenarioMetric(url string, itemName string) map[string]int {
-	stats := make(map[string]int)
-
-	stats["instantiation"] = 0
-	stats["curr_count"] = 0
-	stats["overflow"] = 0
-	stats["pour"] = 0
-	stats["underflow"] = 0
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_bucket_created_total":
-				stats["instantiation"] += ival
-			case "cs_buckets":
-				stats["curr_count"] += ival
-			case "cs_bucket_overflowed_total":
-				stats["overflow"] += ival
-			case "cs_bucket_poured_total":
-				stats["pour"] += ival
-			case "cs_bucket_underflowed_total":
-				stats["underflow"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetPrometheusMetric(url string) []*prom2json.Family {
-	mfChan := make(chan *dto.MetricFamily, 1024)
-
-	// Start with the DefaultTransport for sane defaults.
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	// Conservatively disable HTTP keep-alives as this program will only
-	// ever need a single HTTP request.
-	transport.DisableKeepAlives = true
-	// Timeout early if the server doesn't even return the headers.
-	transport.ResponseHeaderTimeout = time.Minute
-
-	go func() {
-		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
-		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
-		if err != nil {
-			log.Fatalf("failed to fetch prometheus metrics : %v", err)
-		}
-	}()
-
-	result := []*prom2json.Family{}
-	for mf := range mfChan {
-		result = append(result, prom2json.NewFamily(mf))
-	}
-	log.Debugf("Finished reading prometheus output, %d entries", len(result))
-
-	return result
-}
-
-type unit struct {
-	value  int64
-	symbol string
-}
-
-var ranges = []unit{
-	{value: 1e18, symbol: "E"},
-	{value: 1e15, symbol: "P"},
-	{value: 1e12, symbol: "T"},
-	{value: 1e9,  symbol: "G"},
-	{value: 1e6,  symbol: "M"},
-	{value: 1e3,  symbol: "k"},
-	{value: 1,    symbol: ""},
-}
-
-func formatNumber(num int) string {
-	goodUnit := unit{}
-	for _, u := range ranges {
-		if int64(num) >= u.value {
-			goodUnit = u
-			break
-		}
-	}
-
-	if goodUnit.value == 1 {
-		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
-	}
-
-	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
-	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
-}
-
 func getDBClient() (*database.Client, error) {
 	var err error
 	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
@@ -518,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
 	}
 
 	return slice
-
 }

cmd/crowdsec-cli/utils_table.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"strconv"
 
 	"github.com/aquasecurity/table"
 	"github.com/enescakir/emoji"
@@ -10,14 +11,15 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
+func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
 	t := newLightTable(out)
 	t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	for _, status := range statuses {
-		t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
+	for _, item := range items {
+		status, emo := item.InstallStatus()
+		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.State.LocalVersion, item.State.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()
@@ -31,11 +33,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
 	t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
 
 	t.AddRow(
-		fmt.Sprintf("%d", metrics["curr_count"]),
-		fmt.Sprintf("%d", metrics["overflow"]),
-		fmt.Sprintf("%d", metrics["instantiation"]),
-		fmt.Sprintf("%d", metrics["pour"]),
-		fmt.Sprintf("%d", metrics["underflow"]),
+		strconv.Itoa(metrics["curr_count"]),
+		strconv.Itoa(metrics["overflow"]),
+		strconv.Itoa(metrics["instantiation"]),
+		strconv.Itoa(metrics["pour"]),
+		strconv.Itoa(metrics["underflow"]),
 	)
 
 	renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@@ -43,23 +45,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
 }
 
 func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
-	skip := true
 	t := newTable(out)
 	t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
 
+	// don't show table if no hits
+	showTable := false
+
 	for source, stats := range metrics {
 		if stats["hits"] > 0 {
 			t.AddRow(
 				source,
-				fmt.Sprintf("%d", stats["hits"]),
-				fmt.Sprintf("%d", stats["parsed"]),
-				fmt.Sprintf("%d", stats["unparsed"]),
+				strconv.Itoa(stats["hits"]),
+				strconv.Itoa(stats["parsed"]),
+				strconv.Itoa(stats["unparsed"]),
 			)
-			skip = false
+			showTable = true
 		}
 	}
 
-	if !skip {
+	if showTable {
 		renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
 		t.Render()
 	}

cmd/crowdsec/crowdsec.go

@@ -20,21 +20,16 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
+func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
 	var err error
 
-	// Populate cwhub package tools
-	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
-		return nil, fmt.Errorf("while loading hub index: %w", err)
-	}
-
 	// Start loading configs
-	csParsers := parser.NewParsers()
+	csParsers := parser.NewParsers(hub)
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
 		return nil, fmt.Errorf("while loading parsers: %w", err)
 	}
 
-	if err := LoadBuckets(cConfig); err != nil {
+	if err := LoadBuckets(cConfig, hub); err != nil {
 		return nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 
@@ -44,7 +39,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	return csParsers, nil
 }
 
-func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
+func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
 	inputEventChan = make(chan types.Event)
 	inputLineChan = make(chan types.Event)
 
@@ -99,7 +94,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 		for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
 			outputsTomb.Go(func() error {
 				defer trace.CatchPanic("crowdsec/runOutput")
-				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
+				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
 					log.Fatalf("starting outputs error : %s", err)
 					return err
 				}
@@ -131,7 +126,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 	return nil
 }
 
-func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
+func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
 	crowdsecTomb.Go(func() error {
 		defer trace.CatchPanic("crowdsec/serveCrowdsec")
 		go func() {
@@ -139,7 +134,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
 			// this logs every time, even at config reload
 			log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
 			agentReady <- true
-			if err := runCrowdsec(cConfig, parsers); err != nil {
+			if err := runCrowdsec(cConfig, parsers, hub); err != nil {
 				log.Fatalf("unable to start crowdsec routines: %s", err)
 			}
 		}()

cmd/crowdsec/main.go

@@ -75,20 +75,20 @@ type Flags struct {
 
 type labelsMap map[string]string
 
-func LoadBuckets(cConfig *csconfig.Config) error {
+func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
 	var (
 		err   error
 		files []string
 	)
-	for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
-		if hubScenarioItem.Installed {
-			files = append(files, hubScenarioItem.LocalPath)
+	for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
+		if hubScenarioItem.State.Installed {
+			files = append(files, hubScenarioItem.State.LocalPath)
 		}
 	}
 	buckets = leakybucket.NewBuckets()
 
 	log.Infof("Loading %d scenario files", len(files))
-	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
+	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
 
 	if err != nil {
 		return fmt.Errorf("scenario loading failed: %v", err)
@@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
 	cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
 	if err != nil {
-		return nil, err
-	}
-
-	if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
-		return nil, fmt.Errorf("unable to load configuration: common section is empty")
+		return nil, fmt.Errorf("while loading configuration file: %w", err)
 	}
 
 	cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		dumpStates = true
 	}
 
-	// Configuration paths are dependency to load crowdsec configuration
-	if err := cConfig.LoadConfigurationPaths(); err != nil {
-		return nil, err
-	}
-
 	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
 		// if we're in time-machine mode, we don't want to log to file
 		cConfig.Common.LogMedia = "stdout"

cmd/crowdsec/metrics.go

@@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 	if !config.Enabled {
 		return
 	}
-	if config.ListenAddr == "" {
-		log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
-		config.ListenAddr = "127.0.0.1"
-	}
-	if config.ListenPort == 0 {
-		log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
-		config.ListenPort = 6060
-	}
 
 	// Registering prometheus
 	// If in aggregated mode, do not register events associated with a source, to keep the cardinality low

cmd/crowdsec/output.go

@@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
 var bucketOverflows []types.Event
 
 func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
-	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
+	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
+	apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
 
 	var err error
 	ticker := time.NewTicker(1 * time.Second)
@@ -70,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	var cache []types.RuntimeAlert
 	var cacheMutex sync.Mutex
 
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		return fmt.Errorf("loading list of installed hub scenarios: %w", err)
 	}
@@ -93,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 		URL:            apiURL,
 		PapiURL:        papiURL,
 		VersionPrefix:  "v1",
-		UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
+		UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)},
 	})
 	if err != nil {
 		return fmt.Errorf("new client api: %w", err)

cmd/crowdsec/serve.go

@@ -14,6 +14,7 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 	}
 
 	if !cConfig.DisableAgent {
-		csParsers, err := initCrowdsec(cConfig)
+		hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
+		if err != nil {
+			return nil, fmt.Errorf("while loading hub index: %w", err)
+		}
+
+		csParsers, err := initCrowdsec(cConfig, hub)
 		if err != nil {
 			return nil, fmt.Errorf("unable to init crowdsec: %w", err)
 		}
@@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 		}
 
 		agentReady := make(chan bool, 1)
-		serveCrowdsec(csParsers, cConfig, agentReady)
+		serveCrowdsec(csParsers, cConfig, hub, agentReady)
 	}
 
 	log.Printf("Reload is finished")
@@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
 	}
 
 	if !cConfig.DisableAgent {
-		csParsers, err := initCrowdsec(cConfig)
+		hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
+		if err != nil {
+			return fmt.Errorf("while loading hub index: %w", err)
+		}
+
+		csParsers, err := initCrowdsec(cConfig, hub)
 		if err != nil {
 			return fmt.Errorf("crowdsec init: %w", err)
 		}
 
 		// if it's just linting, we're done
 		if !flags.TestMode {
-			serveCrowdsec(csParsers, cConfig, agentReady)
+			serveCrowdsec(csParsers, cConfig, hub, agentReady)
 		}
 	} else {
 		agentReady <- true

config/config.yaml

@@ -6,7 +6,6 @@ common:
   log_max_size: 20
   compress_logs: true
   log_max_files: 10
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

config/config_win.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/config_win_no_lapi.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/dev.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: true
   log_media: stdout
   log_level: info
-  working_dir: .
 config_paths:
   config_dir: ./config
   data_dir: ./data/   

config/user.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data

docker/config.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

docker/docker_start.sh

@@ -101,19 +101,23 @@ register_bouncer() {
 # $2 can be install, remove, upgrade
 # $3 is a list of object names separated by space
 cscli_if_clean() {
+    local itemtype="$1"
+    local action="$2"
+    local objs=$3
+    shift 3
     # loop over all objects
-    for obj in $3; do
-        if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
-            echo "Object $1/$obj is tainted, skipping"
+    for obj in $objs; do
+        if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
+            echo "Object $itemtype/$obj is tainted, skipping"
         else
 #            # Too verbose? Only show errors if not in debug mode
 #            if [ "$DEBUG" != "true" ]; then
 #                error_only=--error
 #            fi
             error_only=""
-            echo "Running: cscli $error_only $1 $2 \"$obj\""
+            echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
             # shellcheck disable=SC2086
-            cscli $error_only "$1" "$2" "$obj"
+            cscli $error_only "$itemtype" "$action" "$obj" "$@"
         fi
     done
 }
@@ -327,22 +331,22 @@ fi
 ## Remove collections, parsers, scenarios & postoverflows
 if [ "$DISABLE_COLLECTIONS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
+    cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force
 fi
 
 if [ "$DISABLE_PARSERS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean parsers remove "$DISABLE_PARSERS"
+    cscli_if_clean parsers remove "$DISABLE_PARSERS" --force
 fi
 
 if [ "$DISABLE_SCENARIOS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
+    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force
 fi
 
 if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
+    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
 fi
 
 ## Register bouncers via env

docker/test/tests/test_hub_collections.py

@@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor):
         cs.wait_for_log([
             # f'*collections install "{it1}"*'
             # f'*collections install "{it2}"*'
-            f'*Enabled collections : {it1}*',
-            f'*Enabled collections : {it2}*',
+            f'*Enabled collections: {it1}*',
+            f'*Enabled collections: {it2}*',
         ])
 
 
@@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor):
         assert it not in items
         logs = cs.log_lines()
         # check that there was no attempt to install
-        assert not any(f'Enabled collections : {it}' in line for line in logs)
+        assert not any(f'Enabled collections: {it}' in line for line in logs)
 
 
 # already done in bats, prividing here as example of a somewhat complex test
@@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
         # implicit check for tainted=False
         assert items[coll]['status'] == 'enabled'
         cs.wait_for_log([
-            f'*Enabled collections : {coll}*',
+            f'*Enabled collections: {coll}*',
         ])
 
         scenario = 'crowdsecurity/http-crawl-non_statics'

docker/test/tests/test_hub_scenarios.py

@@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor):
     }
     with crowdsec(flavor=flavor, environment=env) as cs:
         cs.wait_for_log([
-            f'*scenarios install "{it1}*"',
-            f'*scenarios install "{it2}*"',
+            f'*scenarios install "{it1}"*',
+            f'*scenarios install "{it2}"*',
             "*Starting processing data*"
         ])
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/c-robinson/iplib v1.0.3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.4
+	github.com/crowdsecurity/go-cs-lib v0.0.5
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -140,8 +140,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
-github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
+github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
+github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

pkg/csconfig/api.go

@@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
 		log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
 	}
 
-	if err := c.LoadCommon(); err != nil {
-		return fmt.Errorf("loading common configuration: %s", err)
-	}
-
 	c.API.Server.LogDir = c.Common.LogDir
 	c.API.Server.LogMedia = c.Common.LogMedia
 	c.API.Server.CompressLogs = c.Common.CompressLogs

pkg/csconfig/api_test.go

@@ -3,7 +3,6 @@ package csconfig
 import (
 	"net"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) {
 	err := tmpLAPI.LoadProfiles()
 	require.NoError(t, err)
 
-	LogDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
 	logLevel := log.InfoLevel
 	config := &Config{}
 	fcontent, err := os.ReadFile("./testdata/config.yaml")
@@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) {
 					DbPath: "./testdata/test.db",
 				},
 				Common: &CommonCfg{
-					LogDir:   "./testdata/",
+					LogDir:   "./testdata",
 					LogMedia: "stdout",
 				},
 				DisableAPI: false,
@@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) {
 					ShareContext:          ptr.Of(false),
 					ConsoleManagement:     ptr.Of(false),
 				},
-				LogDir:   LogDirFullPath,
+				LogDir:   "./testdata",
 				LogMedia: "stdout",
 				OnlineClient: &OnlineApiClientCfg{
 					CredentialsFilePath: "./testdata/online-api-secrets.yaml",

pkg/csconfig/common.go

@@ -14,7 +14,7 @@ type CommonCfg struct {
 	LogMedia       string     `yaml:"log_media"`
 	LogDir         string     `yaml:"log_dir,omitempty"` //if LogMedia = file
 	LogLevel       *log.Level `yaml:"log_level"`
-	WorkingDir     string     `yaml:"working_dir,omitempty"` ///var/run
+	WorkingDir     string     `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
 	CompressLogs   *bool      `yaml:"compress_logs,omitempty"`
 	LogMaxSize     int        `yaml:"log_max_size,omitempty"`
 	LogMaxAge      int        `yaml:"log_max_age,omitempty"`
@@ -22,15 +22,18 @@ type CommonCfg struct {
 	ForceColorLogs bool       `yaml:"force_color_logs,omitempty"`
 }
 
-func (c *Config) LoadCommon() error {
+func (c *Config) loadCommon() error {
 	var err error
 	if c.Common == nil {
-		return fmt.Errorf("no common block provided in configuration file")
+		c.Common = &CommonCfg{}
+	}
+
+	if c.Common.LogMedia == "" {
+		c.Common.LogMedia = "stdout"
 	}
 
 	var CommonCleanup = []*string{
 		&c.Common.LogDir,
-		&c.Common.WorkingDir,
 	}
 	for _, k := range CommonCleanup {
 		if *k == "" {

pkg/csconfig/common_test.go

@@ -1,83 +0,0 @@
-package csconfig
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-)
-
-func TestLoadCommon(t *testing.T) {
-	pidDirPath := "./testdata"
-	LogDirFullPath, err := filepath.Abs("./testdata/log/")
-	require.NoError(t, err)
-
-	WorkingDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	tests := []struct {
-		name        string
-		input       *Config
-		expected    *CommonCfg
-		expectedErr string
-	}{
-		{
-			name: "basic valid configuration",
-			input: &Config{
-				Common: &CommonCfg{
-					Daemonize:  true,
-					PidDir:     "./testdata",
-					LogMedia:   "file",
-					LogDir:     "./testdata/log/",
-					WorkingDir: "./testdata/",
-				},
-			},
-			expected: &CommonCfg{
-				Daemonize:  true,
-				PidDir:     pidDirPath,
-				LogMedia:   "file",
-				LogDir:     LogDirFullPath,
-				WorkingDir: WorkingDirFullPath,
-			},
-		},
-		{
-			name: "empty working dir",
-			input: &Config{
-				Common: &CommonCfg{
-					Daemonize: true,
-					PidDir:    "./testdata",
-					LogMedia:  "file",
-					LogDir:    "./testdata/log/",
-				},
-			},
-			expected: &CommonCfg{
-				Daemonize: true,
-				PidDir:    pidDirPath,
-				LogMedia:  "file",
-				LogDir:    LogDirFullPath,
-			},
-		},
-		{
-			name:        "no common",
-			input:       &Config{},
-			expected:    nil,
-			expectedErr: "no common block provided in configuration file",
-		},
-	}
-
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadCommon()
-			cstest.RequireErrorContains(t, err, tc.expectedErr)
-			if tc.expectedErr != "" {
-				return
-			}
-
-			assert.Equal(t, tc.expected, tc.input.Common)
-		})
-	}
-}

pkg/csconfig/config.go

@@ -36,7 +36,7 @@ type Config struct {
 	PluginConfig *PluginCfg          `yaml:"plugin_config,omitempty"`
 	DisableAPI   bool                `yaml:"-"`
 	DisableAgent bool                `yaml:"-"`
-	Hub          *Hub                `yaml:"-"`
+	Hub          *LocalHubCfg        `yaml:"-"`
 }
 
 func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
@@ -58,6 +58,37 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
 		// this is actually the "merged" yaml
 		return nil, "", fmt.Errorf("%s: %w", configFile, err)
 	}
+
+	if cfg.Prometheus == nil {
+		cfg.Prometheus = &PrometheusCfg{}
+	}
+
+	if cfg.Prometheus.ListenAddr == "" {
+		cfg.Prometheus.ListenAddr = "127.0.0.1"
+		log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr)
+	}
+
+	if cfg.Prometheus.ListenPort == 0 {
+		cfg.Prometheus.ListenPort = 6060
+		log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
+	}
+
+	if err = cfg.loadCommon(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadConfigurationPaths(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadHub(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadCSCLI(); err != nil {
+		return nil, "", err
+	}
+
 	return &cfg, configData, nil
 }
 
@@ -65,11 +96,8 @@ func NewDefaultConfig() *Config {
 	logLevel := log.InfoLevel
 	commonCfg := CommonCfg{
 		Daemonize: false,
-		PidDir:    "/tmp/",
 		LogMedia:  "stdout",
-		//LogDir unneeded
-		LogLevel:   &logLevel,
-		WorkingDir: ".",
+		LogLevel:  &logLevel,
 	}
 	prometheus := PrometheusCfg{
 		Enabled: true,

pkg/csconfig/config_paths.go

@@ -15,7 +15,7 @@ type ConfigurationPaths struct {
 	NotificationDir    string `yaml:"notification_dir,omitempty"`
 }
 
-func (c *Config) LoadConfigurationPaths() error {
+func (c *Config) loadConfigurationPaths() error {
 	var err error
 	if c.ConfigPaths == nil {
 		return fmt.Errorf("no configuration paths provided")

pkg/csconfig/config_test.go

@@ -15,10 +15,10 @@ func TestNormalLoad(t *testing.T) {
 	require.NoError(t, err)
 
 	_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
-	assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
+	require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
 
 	_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
-	assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n  line 1: field simulation not found in type csconfig.Config")
+	require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n  line 1: field simulation not found in type csconfig.Config")
 }
 
 func TestNewCrowdSecConfig(t *testing.T) {

pkg/csconfig/crowdsec_service.go

@@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct {
 	BucketStateDumpDir        string            `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
 	BucketsGCEnabled          bool              `yaml:"-"`                          // we need to garbage collect buckets when in forensic mode
 
-	HubDir             string              `yaml:"-"`
-	DataDir            string              `yaml:"-"`
-	ConfigDir          string              `yaml:"-"`
-	HubIndexFile       string              `yaml:"-"`
 	SimulationFilePath string              `yaml:"-"`
 	ContextToSend      map[string][]string `yaml:"-"`
 }
@@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("load error (simulation): %w", err)
 	}
 
-	c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
-	c.Crowdsec.DataDir = c.ConfigPaths.DataDir
-	c.Crowdsec.HubDir = c.ConfigPaths.HubDir
-	c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
-
 	if c.Crowdsec.ParserRoutinesCount <= 0 {
 		c.Crowdsec.ParserRoutinesCount = 1
 	}
@@ -145,15 +136,11 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("loading api client: %s", err)
 	}
 
-	if err := c.LoadHub(); err != nil {
-		return fmt.Errorf("while loading hub: %w", err)
-	}
-
 	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
 	fallback := false
 	if c.Crowdsec.ConsoleContextPath == "" {
 		// fallback to default config file
-		c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
+		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
 		fallback = true
 	}
 

pkg/csconfig/crowdsec_service_test.go

@@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) {
 	acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
 	require.NoError(t, err)
 
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
 	require.NoError(t, err)
 
@@ -66,10 +54,6 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        "",
 				ConsoleContextPath:        contextFileFullPath,
 				AcquisitionFilePath:       acquisFullPath,
-				ConfigDir:                 configDirFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -109,10 +93,6 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        acquisDirFullPath,
 				AcquisitionFilePath:       acquisFullPath,
 				ConsoleContextPath:        contextFileFullPath,
-				ConfigDir:                 configDirFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -141,7 +121,7 @@ func TestLoadCrowdsec(t *testing.T) {
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					ConsoleContextPath:        contextFileFullPath,
+					ConsoleContextPath:        "./testdata/context.yaml",
 					ConsoleContextValueLength: 10,
 				},
 			},
@@ -149,10 +129,6 @@ func TestLoadCrowdsec(t *testing.T) {
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				AcquisitionFilePath:       "",
-				ConfigDir:                 configDirFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
 				ConsoleContextPath:        contextFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,

pkg/csconfig/cscli.go

@@ -1,5 +1,9 @@
 package csconfig
 
+import (
+	"fmt"
+)
+
 /*cscli specific config, such as hub directory*/
 type CscliCfg struct {
 	Output             string            `yaml:"output,omitempty"`
@@ -7,25 +11,19 @@ type CscliCfg struct {
 	HubBranch          string            `yaml:"hub_branch"`
 	SimulationConfig   *SimulationConfig `yaml:"-"`
 	DbConfig           *DatabaseCfg      `yaml:"-"`
-	HubDir             string            `yaml:"-"`
-	DataDir            string            `yaml:"-"`
-	ConfigDir          string            `yaml:"-"`
-	HubIndexFile       string            `yaml:"-"`
+
 	SimulationFilePath string            `yaml:"-"`
 	PrometheusUrl      string            `yaml:"prometheus_uri"`
 }
 
-func (c *Config) LoadCSCLI() error {
+func (c *Config) loadCSCLI() error {
 	if c.Cscli == nil {
 		c.Cscli = &CscliCfg{}
 	}
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
+
+	if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
+		c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
 	}
-	c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
-	c.Cscli.DataDir = c.ConfigPaths.DataDir
-	c.Cscli.HubDir = c.ConfigPaths.HubDir
-	c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
 
 	return nil
 }

pkg/csconfig/cscli_test.go

@@ -1,28 +1,14 @@
 package csconfig
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadCSCLI(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
@@ -38,26 +24,23 @@ func TestLoadCSCLI(t *testing.T) {
 					HubDir:       "./hub",
 					HubIndexFile: "./hub/.index.json",
 				},
+				Prometheus: &PrometheusCfg{
+					Enabled:    true,
+					Level:      "full",
+					ListenAddr: "127.0.0.1",
+					ListenPort: 6060,
+				},
 			},
 			expected: &CscliCfg{
-				ConfigDir:    configDirFullPath,
-				DataDir:      dataFullPath,
-				HubDir:       hubFullPath,
-				HubIndexFile: hubIndexFileFullPath,
+				PrometheusUrl: "http://127.0.0.1:6060/metrics",
 			},
 		},
-		{
-			name:        "no configuration path",
-			input:       &Config{},
-			expected:    &CscliCfg{},
-			expectedErr: "no configuration paths provided",
-		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadCSCLI()
+			err := tc.input.loadCSCLI()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return

pkg/csconfig/hub.go

@@ -1,19 +1,15 @@
 package csconfig
 
-/*cscli specific config, such as hub directory*/
-type Hub struct {
-	HubIndexFile   string
-	HubDir         string
-	InstallDir     string
-	InstallDataDir string
+// LocalHubCfg holds the configuration for a local hub: where to download etc.
+type LocalHubCfg struct {
+	HubIndexFile   string	// Path to the local index file
+	HubDir         string	// Where the hub items are downloaded
+	InstallDir     string	// Where to install items
+	InstallDataDir string	// Where to install data
 }
 
-func (c *Config) LoadHub() error {
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
-
-	c.Hub = &Hub{
+func (c *Config) loadHub() error {
+	c.Hub = &LocalHubCfg{
 		HubIndexFile:   c.ConfigPaths.HubIndexFile,
 		HubDir:         c.ConfigPaths.HubDir,
 		InstallDir:     c.ConfigPaths.ConfigDir,

pkg/csconfig/hub_test.go

@@ -1,32 +1,18 @@
 package csconfig
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadHub(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
-		expected    *Hub
+		expected    *LocalHubCfg
 		expectedErr string
 	}{
 		{
@@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) {
 					HubIndexFile: "./hub/.index.json",
 				},
 			},
-			expected: &Hub{
-				HubDir:         hubFullPath,
-				HubIndexFile:   hubIndexFileFullPath,
-				InstallDir:     configDirFullPath,
-				InstallDataDir: dataFullPath,
+			expected: &LocalHubCfg{
+				HubDir:         "./hub",
+				HubIndexFile:   "./hub/.index.json",
+				InstallDir:     "./testdata",
+				InstallDataDir: "./data",
 			},
 		},
-		{
-			name: "no data dir",
-			input: &Config{
-				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./testdata",
-					HubDir:       "./hub",
-					HubIndexFile: "./hub/.index.json",
-				},
-			},
-			expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
-		},
-		{
-			name:        "no configuration path",
-			input:       &Config{},
-			expectedErr: "no configuration paths provided",
-		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadHub()
+			err := tc.input.loadHub()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return

pkg/csconfig/prometheus.go

@@ -1,19 +1,8 @@
 package csconfig
 
-import "fmt"
-
 type PrometheusCfg struct {
 	Enabled    bool   `yaml:"enabled"`
 	Level      string `yaml:"level"` //aggregated|full
 	ListenAddr string `yaml:"listen_addr"`
 	ListenPort int    `yaml:"listen_port"`
 }
-
-func (c *Config) LoadPrometheus() error {
-	if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil {
-		if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
-			c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
-		}
-	}
-	return nil
-}

pkg/csconfig/prometheus_test.go

@@ -1,42 +0,0 @@
-package csconfig
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-)
-
-func TestLoadPrometheus(t *testing.T) {
-	tests := []struct {
-		name        string
-		input       *Config
-		expectedURL string
-		expectedErr string
-	}{
-		{
-			name: "basic valid configuration",
-			input: &Config{
-				Prometheus: &PrometheusCfg{
-					Enabled:    true,
-					Level:      "full",
-					ListenAddr: "127.0.0.1",
-					ListenPort: 6060,
-				},
-				Cscli: &CscliCfg{},
-			},
-			expectedURL: "http://127.0.0.1:6060",
-		},
-	}
-
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadPrometheus()
-			cstest.RequireErrorContains(t, err, tc.expectedErr)
-
-			require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl)
-		})
-	}
-}

pkg/csconfig/simulation.go

@@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
 }
 
 func (c *Config) LoadSimulation() error {
-
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
-
 	simCfg := SimulationConfig{}
 	if c.ConfigPaths.SimulationFilePath == "" {
 		c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")

pkg/csconfig/simulation_test.go

@@ -2,7 +2,6 @@ package csconfig
 
 import (
 	"fmt"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -12,12 +11,6 @@ import (
 )
 
 func TestSimulationLoading(t *testing.T) {
-	testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml")
-	require.NoError(t, err)
-
-	badYamlFullPath, err := filepath.Abs("./testdata/config.yaml")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
@@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage),
+			expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage),
 		},
 		{
 			name: "basic bad file content",
@@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+			expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
 		},
 		{
 			name: "basic bad file content",
@@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+			expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
 		},
 	}
 

pkg/csconfig/testdata/config.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: false
   log_media: stdout
   log_level: info
-  working_dir: .
 prometheus:
   enabled: true
   level: full

pkg/cwhub/cwhub.go

@@ -2,287 +2,31 @@ package cwhub
 
 import (
 	"fmt"
-	"os"
+	"net/http"
 	"path/filepath"
-	"sort"
 	"strings"
-
-	"github.com/enescakir/emoji"
-	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/mod/semver"
-)
-
-const (
-	HubIndexFile = ".index.json"
-
-	// managed item types
-	PARSERS       = "parsers"
-	PARSERS_OVFLW = "postoverflows"
-	SCENARIOS     = "scenarios"
-	COLLECTIONS   = "collections"
+	"time"
 )
 
-var (
-	ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS}
-
-	ErrMissingReference = errors.New("Reference(s) missing in collection")
-
-	// XXX: can we remove these globals?
-	skippedLocal       = 0
-	skippedTainted     = 0
-	RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
-	HubBranch          = "master"
-	hubIdx             map[string]map[string]Item
-)
-
-type ItemVersion struct {
-	Digest     string `json:"digest,omitempty"` // meow
-	Deprecated bool   `json:"deprecated,omitempty"`
-}
-
-type ItemHubStatus struct {
-	Name         string `json:"name"`
-	LocalVersion string `json:"local_version"`
-	LocalPath    string `json:"local_path"`
-	Description  string `json:"description"`
-	UTF8Status   string `json:"utf8_status"`
-	Status       string `json:"status"`
-}
-
-// Item can be: parser, scenario, collection..
-type Item struct {
-	// descriptive info
-	Type                 string   `json:"type,omitempty"                   yaml:"type,omitempty"`                   // parser|postoverflows|scenario|collection(|enrich)
-	Stage                string   `json:"stage,omitempty"                  yaml:"stage,omitempty"`                  // Stage for parser|postoverflow: s00-raw/s01-...
-	Name                 string   `json:"name,omitempty"`                                                           // as seen in .config.json, usually "author/name"
-	FileName             string   `json:"file_name,omitempty"`                                                      // the filename, ie. apache2-logs.yaml
-	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .config.json
-	Author               string   `json:"author,omitempty"`                                                         // as seen in .config.json
-	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`             // as seen in .config.json
-	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
-
-	// remote (hub) info
-	RemotePath string                 `json:"path,omitempty"      yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
-	Version    string                 `json:"version,omitempty"`                                // the last version
-	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                     // the list of existing versions
-
-	// local (deployed) info
-	LocalPath    string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
-	LocalVersion string `json:"local_version,omitempty"`
-	LocalHash    string `json:"local_hash,omitempty"` // the local meow
-	Installed    bool   `json:"installed,omitempty"`
-	Downloaded   bool   `json:"downloaded,omitempty"`
-	UpToDate     bool   `json:"up_to_date,omitempty"`
-	Tainted      bool   `json:"tainted,omitempty"` // has it been locally modified
-	Local        bool   `json:"local,omitempty"`   // if it's a non versioned control one
-
-	// if it's a collection, it's not a single file
-	Parsers       []string `json:"parsers,omitempty"       yaml:"parsers,omitempty"`
-	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
-	Scenarios     []string `json:"scenarios,omitempty"     yaml:"scenarios,omitempty"`
-	Collections   []string `json:"collections,omitempty"   yaml:"collections,omitempty"`
-}
-
-func (i *Item) status() (string, emoji.Emoji) {
-	status := "disabled"
-	ok := false
-
-	if i.Installed {
-		ok = true
-		status = "enabled"
-	}
-
-	managed := true
-	if i.Local {
-		managed = false
-		status += ",local"
-	}
-
-	warning := false
-	if i.Tainted {
-		warning = true
-		status += ",tainted"
-	} else if !i.UpToDate && !i.Local {
-		warning = true
-		status += ",update-available"
-	}
-
-	emo := emoji.QuestionMark
-
-	switch {
-	case !managed:
-		emo = emoji.House
-	case !i.Installed:
-		emo = emoji.Prohibited
-	case warning:
-		emo = emoji.Warning
-	case ok:
-		emo = emoji.CheckMark
-	}
-
-	return status, emo
-}
-
-func (i *Item) hubStatus() ItemHubStatus {
-	status, emo := i.status()
-
-	return ItemHubStatus{
-		Name:         i.Name,
-		LocalVersion: i.LocalVersion,
-		LocalPath:    i.LocalPath,
-		Description:  i.Description,
-		Status:       status,
-		UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
-	}
-}
-
-// versionStatus: semver requires 'v' prefix
-func (i *Item) versionStatus() int {
-	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
-}
-
-func GetItemMap(itemType string) map[string]Item {
-	m, ok := hubIdx[itemType]
-	if !ok {
-		return nil
-	}
-
-	return m
-}
-
-// Given a FileInfo, extract the map key. Follow a symlink if necessary
-func itemKey(itemPath string) (string, error) {
-	f, err := os.Lstat(itemPath)
-	if err != nil {
-		return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
-	}
-
-	if f.Mode()&os.ModeSymlink == 0 {
-		// it's not a symlink, so the filename itsef should be the key
-		return filepath.Base(itemPath), nil
-	}
-
-	// resolve the symlink to hub file
-	pathInHub, err := os.Readlink(itemPath)
-	if err != nil {
-		return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
-	}
-
-	author := filepath.Base(filepath.Dir(pathInHub))
-
-	fname := filepath.Base(pathInHub)
-	fname = strings.TrimSuffix(fname, ".yaml")
-	fname = strings.TrimSuffix(fname, ".yml")
-
-	return fmt.Sprintf("%s/%s", author, fname), nil
+var hubClient = &http.Client{
+	Timeout: 120 * time.Second,
 }
 
-// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
-func GetItemByPath(itemType string, itemPath string) (*Item, error) {
-	itemKey, err := itemKey(itemPath)
+// safePath returns a joined path and ensures that it does not escape the base directory.
+func safePath(dir, filePath string) (string, error) {
+	absBaseDir, err := filepath.Abs(filepath.Clean(dir))
 	if err != nil {
-		return nil, err
-	}
-
-	m := GetItemMap(itemType)
-	if m == nil {
-		return nil, fmt.Errorf("item type %s doesn't exist", itemType)
-	}
-
-	v, ok := m[itemKey]
-	if !ok {
-		return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
-	}
-
-	return &v, nil
-}
-
-func GetItem(itemType string, itemName string) *Item {
-	if m, ok := GetItemMap(itemType)[itemName]; ok {
-		return &m
-	}
-
-	return nil
-}
-
-func AddItem(itemType string, item Item) error {
-	for _, itype := range ItemTypes {
-		if itype == itemType {
-			hubIdx[itemType][item.Name] = item
-			return nil
-		}
-	}
-
-	return fmt.Errorf("ItemType %s is unknown", itemType)
-}
-
-func DisplaySummary() {
-	log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
-		len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
-
-	if skippedLocal > 0 || skippedTainted > 0 {
-		log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
-	}
-}
-
-func GetInstalledItems(itemType string) ([]Item, error) {
-	items, ok := hubIdx[itemType]
-	if !ok {
-		return nil, fmt.Errorf("no %s in hubIdx", itemType)
+		return "", err
 	}
 
-	retItems := make([]Item, 0)
-
-	for _, item := range items {
-		if item.Installed {
-			retItems = append(retItems, item)
-		}
-	}
-
-	return retItems, nil
-}
-
-func GetInstalledItemsAsString(itemType string) ([]string, error) {
-	items, err := GetInstalledItems(itemType)
+	absFilePath, err := filepath.Abs(filepath.Join(dir, filePath))
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	retStr := make([]string, len(items))
-
-	for i, it := range items {
-		retStr[i] = it.Name
-	}
-
-	return retStr, nil
-}
-
-// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
-func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
-	if _, ok := hubIdx[itemType]; !ok {
-		log.Errorf("type %s doesn't exist", itemType)
-
-		return nil
+	if !strings.HasPrefix(absFilePath, absBaseDir) {
+		return "", fmt.Errorf("path %s escapes base directory %s", filePath, dir)
 	}
 
-	ret := make([]ItemHubStatus, 0)
-
-	// remember, you do it for the user :)
-	for _, item := range hubIdx[itemType] {
-		if name != "" && name != item.Name {
-			// user has requested a specific name
-			continue
-		}
-		// Only enabled items ?
-		if !all && !item.Installed {
-			continue
-		}
-		// Check the item status
-		ret = append(ret, item.hubStatus())
-	}
-
-	sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
-
-	return ret
+	return absFilePath, nil
 }

pkg/cwhub/cwhub_test.go

@@ -9,14 +9,13 @@ import (
 	"testing"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
+const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
+
 /*
  To test :
   - Download 'first' hub index
@@ -28,294 +27,63 @@ import (
 
 var responseByPath map[string]string
 
-func TestItemStatus(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
-
-	// DownloadHubIdx()
-	err := UpdateHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to download index")
-
-	err = GetHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to load hub index")
-
-	// get existing map
-	x := GetItemMap(COLLECTIONS)
-	require.NotEmpty(t, x)
-
-	// Get item : good and bad
-	for k := range x {
-		item := GetItem(COLLECTIONS, k)
-		require.NotNil(t, item)
-
-		item.Installed = true
-		item.UpToDate = false
-		item.Local = false
-		item.Tainted = false
-
-		txt, _ := item.status()
-		require.Equal(t, "enabled,update-available", txt)
-
-		item.Installed = false
-		item.UpToDate = false
-		item.Local = true
-		item.Tainted = false
+// testHub initializes a temporary hub with an empty json file, optionally updating it.
+func testHub(t *testing.T, update bool) *Hub {
+	tmpDir, err := os.MkdirTemp("", "testhub")
+	require.NoError(t, err)
 
-		txt, _ = item.status()
-		require.Equal(t, "disabled,local", txt)
+	local := &csconfig.LocalHubCfg{
+		HubDir:         filepath.Join(tmpDir, "crowdsec", "hub"),
+		HubIndexFile:   filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"),
+		InstallDir:     filepath.Join(tmpDir, "crowdsec"),
+		InstallDataDir: filepath.Join(tmpDir, "installed-data"),
 	}
 
-	DisplaySummary()
-}
-
-func TestGetters(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
-
-	// DownloadHubIdx()
-	err := UpdateHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to download index")
-
-	err = GetHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to load hub index")
-
-	// get non existing map
-	empty := GetItemMap("ratata")
-	require.Nil(t, empty)
-
-	// get existing map
-	x := GetItemMap(COLLECTIONS)
-	require.NotEmpty(t, x)
+	err = os.MkdirAll(local.HubDir, 0o700)
+	require.NoError(t, err)
 
-	// Get item : good and bad
-	for k := range x {
-		empty := GetItem(COLLECTIONS, k+"nope")
-		require.Nil(t, empty)
+	err = os.MkdirAll(local.InstallDir, 0o700)
+	require.NoError(t, err)
 
-		item := GetItem(COLLECTIONS, k)
-		require.NotNil(t, item)
+	err = os.MkdirAll(local.InstallDataDir, 0o700)
+	require.NoError(t, err)
 
-		// Add item and get it
-		item.Name += "nope"
-		err := AddItem(COLLECTIONS, *item)
-		require.NoError(t, err)
+	err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644)
+	require.NoError(t, err)
 
-		newitem := GetItem(COLLECTIONS, item.Name)
-		require.NotNil(t, newitem)
+	t.Cleanup(func() {
+		os.RemoveAll(tmpDir)
+	})
 
-		err = AddItem("ratata", *item)
-		cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
+	remote := &RemoteHubCfg{
+		Branch:      "master",
+		URLTemplate: mockURLTemplate,
+		IndexPath:   ".index.json",
 	}
-}
-
-func TestIndexDownload(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
 
-	// DownloadHubIdx()
-	err := UpdateHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to download index")
-
-	err = GetHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to load hub index")
-}
-
-func getTestCfg() *csconfig.Config {
-	cfg := &csconfig.Config{Hub: &csconfig.Hub{}}
-	cfg.Hub.InstallDir, _ = filepath.Abs("./install")
-	cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
-	cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
+	hub, err := NewHub(local, remote, update)
+	require.NoError(t, err)
 
-	return cfg
+	return hub
 }
 
-func envSetup(t *testing.T) *csconfig.Config {
-	resetResponseByPath()
+// envSetup initializes the temporary hub and mocks the http client.
+func envSetup(t *testing.T) *Hub {
+	setResponseByPath()
 	log.SetLevel(log.DebugLevel)
 
-	cfg := getTestCfg()
-
-	defaultTransport := http.DefaultClient.Transport
+	defaultTransport := hubClient.Transport
 
 	t.Cleanup(func() {
-		http.DefaultClient.Transport = defaultTransport
+		hubClient.Transport = defaultTransport
 	})
 
 	// Mock the http client
-	http.DefaultClient.Transport = newMockTransport()
+	hubClient.Transport = newMockTransport()
 
-	err := os.MkdirAll(cfg.Hub.InstallDir, 0700)
-	require.NoError(t, err)
+	hub := testHub(t, true)
 
-	err = os.MkdirAll(cfg.Hub.HubDir, 0700)
-	require.NoError(t, err)
-
-	err = UpdateHubIdx(cfg.Hub)
-	require.NoError(t, err)
-
-	// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
-	// 	log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
-	// }
-	// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
-	// 	log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
-	// }
-	return cfg
-}
-
-func envTearDown(cfg *csconfig.Config) {
-	if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
-		log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
-	}
-
-	if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
-		log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
-	}
-}
-
-func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	// Install the parser
-	err := DownloadLatest(cfg, &item, false, false)
-	require.NoError(t, err, "failed to download %s", item.Name)
-
-	_, err = LocalSync(cfg)
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
-
-	err = EnableItem(cfg, &item)
-	require.NoError(t, err, "failed to enable %s", item.Name)
-
-	_, err = LocalSync(cfg)
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
-}
-
-func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
-
-	f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
-	require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
-
-	defer f.Close()
-
-	_, err = f.WriteString("tainted")
-	require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
-
-	// Local sync and check status
-	_, err = LocalSync(cfg)
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
-}
-
-func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
-
-	// Update it + check status
-	err := DownloadLatest(cfg, &item, true, true)
-	require.NoError(t, err, "failed to update %s", item.Name)
-
-	// Local sync and check status
-	_, err = LocalSync(cfg)
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
-}
-
-func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
-
-	// Remove
-	err := DisableItem(cfg, &item, false, false)
-	require.NoError(t, err, "failed to disable %s", item.Name)
-
-	// Local sync and check status
-	warns, err := LocalSync(cfg)
-	require.NoError(t, err, "failed to run localSync")
-	require.Empty(t, warns, "unexpected warnings : %+v", warns)
-
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
-
-	// Purge
-	err = DisableItem(cfg, &item, true, false)
-	require.NoError(t, err, "failed to purge %s", item.Name)
-
-	// Local sync and check status
-	warns, err = LocalSync(cfg)
-	require.NoError(t, err, "failed to run localSync")
-	require.Empty(t, warns, "unexpected warnings : %+v", warns)
-
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
-}
-
-func TestInstallParser(t *testing.T) {
-	/*
-	 - install a random parser
-	 - check its status
-	 - taint it
-	 - check its status
-	 - force update it
-	 - check its status
-	 - remove it
-	*/
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
-
-	getHubIdxOrFail(t)
-	// map iteration is random by itself
-	for _, it := range hubIdx[PARSERS] {
-		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
-		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
-		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
-		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
-		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
-		testDisableItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
-
-		break
-	}
-}
-
-func TestInstallCollection(t *testing.T) {
-	/*
-	 - install a random parser
-	 - check its status
-	 - taint it
-	 - check its status
-	 - force update it
-	 - check its status
-	 - remove it
-	*/
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
-
-	getHubIdxOrFail(t)
-	// map iteration is random by itself
-	for _, it := range hubIdx[COLLECTIONS] {
-		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
-		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
-		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
-		testDisableItem(cfg.Hub, t, it)
-
-		it = hubIdx[COLLECTIONS][it.Name]
-		x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
-		log.Infof("%+v", x)
-
-		break
-	}
+	return hub
 }
 
 type mockTransport struct{}
@@ -324,7 +92,7 @@ func newMockTransport() http.RoundTripper {
 	return &mockTransport{}
 }
 
-// Implement http.RoundTripper
+// Implement http.RoundTripper.
 func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	// Create mocked http.Response
 	response := &http.Response{
@@ -362,7 +130,7 @@ func fileToStringX(path string) string {
 	return strings.ReplaceAll(string(data), "\r\n", "\n")
 }
 
-func resetResponseByPath() {
+func setResponseByPath() {
 	responseByPath = map[string]string{
 		"/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml":    fileToStringX("./testdata/foobar_parser.yaml"),
 		"/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),

pkg/cwhub/dataset.go

@@ -1,70 +1,84 @@
 package cwhub
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
-	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
+// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
 type DataSet struct {
-	Data []*types.DataSource `yaml:"data,omitempty"`
+	Data []types.DataSource `yaml:"data,omitempty"`
 }
 
+// downloadFile downloads a file and writes it to disk, with no hash verification.
 func downloadFile(url string, destPath string) error {
 	log.Debugf("downloading %s in %s", url, destPath)
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
+	resp, err := hubClient.Get(url)
 	if err != nil {
-		return err
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
+		return fmt.Errorf("while downloading %s: %w", url, err)
 	}
 	defer resp.Body.Close()
 
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body))
+		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
 	}
 
-	file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	file, err := os.Create(destPath)
 	if err != nil {
 		return err
 	}
+	defer file.Close()
 
-	_, err = file.Write(body)
+	// avoid reading the whole file in memory
+	_, err = io.Copy(file, resp.Body)
 	if err != nil {
 		return err
 	}
 
-	err = file.Sync()
-	if err != nil {
+	if err = file.Sync(); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func GetData(data []*types.DataSource, dataDir string) error {
-	for _, dataS := range data {
-		destPath := filepath.Join(dataDir, dataS.DestPath)
-		log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
+// downloadDataSet downloads all the data files for an item.
+func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
+	dec := yaml.NewDecoder(reader)
+
+	for {
+		data := &DataSet{}
+
+		if err := dec.Decode(data); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+
+			return fmt.Errorf("while reading file: %w", err)
+		}
+
+		for _, dataS := range data.Data {
+			destPath, err := safePath(dataFolder, dataS.DestPath)
+			if err != nil {
+				return err
+			}
+
+			if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
+				log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
 
-		err := downloadFile(dataS.SourceURL, destPath)
-		if err != nil {
-			return err
+				if err := downloadFile(dataS.SourceURL, destPath); err != nil {
+					return fmt.Errorf("while getting data: %w", err)
+				}
+			}
 		}
 	}
 

pkg/cwhub/dataset_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/jarcoal/httpmock"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDownloadFile(t *testing.T) {
@@ -14,12 +15,14 @@ func TestDownloadFile(t *testing.T) {
 
 	httpmock.Activate()
 	defer httpmock.DeactivateAndReset()
+
 	//OK
 	httpmock.RegisterResponder(
 		"GET",
 		"https://example.com/xx",
 		httpmock.NewStringResponder(200, "example content oneoneone"),
 	)
+
 	httpmock.RegisterResponder(
 		"GET",
 		"https://example.com/x",
@@ -27,17 +30,21 @@ func TestDownloadFile(t *testing.T) {
 	)
 
 	err := downloadFile("https://example.com/xx", examplePath)
-	assert.NoError(t, err)
+	require.NoError(t, err)
+
 	content, err := os.ReadFile(examplePath)
 	assert.Equal(t, "example content oneoneone", string(content))
-	assert.NoError(t, err)
+	require.NoError(t, err)
+
 	//bad uri
 	err = downloadFile("https://zz.com", examplePath)
-	assert.Error(t, err)
+	require.Error(t, err)
+
 	//404
 	err = downloadFile("https://example.com/x", examplePath)
-	assert.Error(t, err)
+	require.Error(t, err)
+
 	//bad target
 	err = downloadFile("https://example.com/xx", "")
-	assert.Error(t, err)
+	require.Error(t, err)
 }

pkg/cwhub/doc.go

@@ -0,0 +1,113 @@
+// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec.
+//
+// # Definitions
+//
+//  - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items.
+//  - The hub INDEX is a JSON file that contains a tree of available hub items.
+//  - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version.
+//  - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items.
+//
+// Once downloaded, hub items can be installed by linking to them from the configuration directory.
+// If an item is present in the configuration directory but it's not a link to the local hub, it is
+// considered as a LOCAL ITEM and won't be removed or upgraded.
+//
+// # Directory Structure
+//
+// A typical directory layout is the following:
+//
+// For the local hub (HubDir = /etc/crowdsec/hub):
+//
+//  - /etc/crowdsec/hub/.index.json
+//  - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
+//  - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
+//
+// For the configuration directory (InstallDir = /etc/crowdsec):
+//
+//  - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
+//  - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
+//  - /etc/crowdsec/scenarios/local-scenario.yaml
+//
+// Note that installed items are not grouped by author, this may change in the future if we want to
+// support items with the same name from different authors.
+//
+// Only parsers and postoverflows have the concept of stage.
+//
+// Additionally, an item can reference a DATA SET that is installed in a different location than
+// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data).
+//
+//  - /var/lib/crowdsec/data/http_path_traversal.txt
+//  - /var/lib/crowdsec/data/jira_cve_2021-26086.txt
+//  - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt
+//  - /var/lib/crowdsec/data/sensitive_data.txt
+//
+//
+// # Using the package
+//
+// The main entry point is the Hub struct. You can create a new instance with NewHub().
+// This constructor takes three parameters, but only the LOCAL HUB configuration is required:
+//
+//	import (
+//		"fmt"
+//		"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+//		"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+//	)
+//
+//	localHub := csconfig.LocalHubCfg{
+//		HubIndexFile:	"/etc/crowdsec/hub/.index.json",
+//		HubDir:		"/etc/crowdsec/hub",
+//		InstallDir:	"/etc/crowdsec",
+//		InstallDataDir: "/var/lib/crowdsec/data",
+//	}
+//	hub, err := cwhub.NewHub(localHub, nil, false)
+//	if err != nil {
+//		return fmt.Errorf("unable to initialize hub: %w", err)
+//	}
+//
+// Now you can use the hub to access the existing items:
+//
+//	// list all the parsers
+//	for _, parser := range hub.GetItemMap(cwhub.PARSERS) {
+//		fmt.Printf("parser: %s\n", parser.Name)
+//	}
+//
+//	// retrieve a specific collection
+//	coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux")
+//	if coll == nil {
+//		return fmt.Errorf("collection not found")
+//	}
+//
+// You can also install items if they have already been downloaded:
+//
+// 	// install a parser
+// 	force := false
+// 	downloadOnly := false
+// 	err := parser.Install(force, downloadOnly)
+// 	if err != nil {
+// 		return fmt.Errorf("unable to install parser: %w", err)
+// 	}
+//
+// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash
+// does not correspond to the latest version available in the index), a download will be attempted and you'll
+// get the error "remote hub configuration is not provided".
+//
+// To provide the remote hub configuration, use the second parameter of NewHub():
+//
+// 	remoteHub := cwhub.RemoteHubCfg{
+//		URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
+//		Branch: "master",
+//		IndexPath: ".index.json",
+//	}
+//	updateIndex := false
+//	hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex)
+//	if err != nil {
+//		return fmt.Errorf("unable to initialize hub: %w", err)
+//	}
+//
+// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
+// placeholders: the branch and the file path (it will be an index or an item).
+//
+// Setting the third parameter to true will download the latest version of the index, if available on the
+// specified branch.
+// There is no exported method to update the index once the hub struct is created.
+//
+package cwhub

pkg/cwhub/download.go

@@ -1,324 +0,0 @@
-package cwhub
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-)
-
-var ErrIndexNotFound = fmt.Errorf("index not found")
-
-func UpdateHubIdx(hub *csconfig.Hub) error {
-	bidx, err := DownloadHubIdx(hub)
-	if err != nil {
-		return fmt.Errorf("failed to download index: %w", err)
-	}
-
-	ret, err := LoadPkgIndex(bidx)
-	if err != nil {
-		if !errors.Is(err, ErrMissingReference) {
-			return fmt.Errorf("failed to read index: %w", err)
-		}
-	}
-
-	hubIdx = ret
-
-	if _, err := LocalSync(hub); err != nil {
-		return fmt.Errorf("failed to sync: %w", err)
-	}
-
-	return nil
-}
-
-func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
-	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
-
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to build request for hub index: %w", err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("failed http request for hub index: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		if resp.StatusCode == http.StatusNotFound {
-			return nil, ErrIndexNotFound
-		}
-
-		return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
-	}
-
-	oldContent, err := os.ReadFile(hub.HubIndexFile)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			log.Warningf("failed to read hub index: %s", err)
-		}
-	} else if bytes.Equal(body, oldContent) {
-		log.Info("hub index is up to date")
-		// write it anyway, can't hurt
-	}
-
-	file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-
-	if err != nil {
-		return nil, fmt.Errorf("while opening hub index file: %w", err)
-	}
-	defer file.Close()
-
-	wsize, err := file.Write(body)
-	if err != nil {
-		return nil, fmt.Errorf("while writing hub index file: %w", err)
-	}
-
-	log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
-
-	return body, nil
-}
-
-// DownloadLatest will download the latest version of Item to the tdir directory
-func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error {
-	var err error
-
-	log.Debugf("Downloading %s %s", target.Type, target.Name)
-
-	if target.Type != COLLECTIONS {
-		if !target.Installed && updateOnly && target.Downloaded {
-			log.Debugf("skipping upgrade of %s : not installed", target.Name)
-			return nil
-		}
-
-		return DownloadItem(hub, target, overwrite)
-	}
-
-	// collection
-	var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
-	for idx, ptr := range tmp {
-		ptrtype := ItemTypes[idx]
-		for _, p := range ptr {
-			val, ok := hubIdx[ptrtype][p]
-			if !ok {
-				return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
-			}
-
-			if !val.Installed && updateOnly && val.Downloaded {
-				log.Debugf("skipping upgrade of %s : not installed", target.Name)
-				continue
-			}
-
-			log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
-			//recurse as it's a collection
-			if ptrtype == COLLECTIONS {
-				log.Tracef("collection, recurse")
-
-				err = DownloadLatest(hub, &val, overwrite, updateOnly)
-				if err != nil {
-					return fmt.Errorf("while downloading %s: %w", val.Name, err)
-				}
-			}
-
-			downloaded := val.Downloaded
-
-			err = DownloadItem(hub, &val, overwrite)
-			if err != nil {
-				return fmt.Errorf("while downloading %s: %w", val.Name, err)
-			}
-
-			// We need to enable an item when it has been added to a collection since latest release of the collection.
-			// We check if val.Downloaded is false because maybe the item has been disabled by the user.
-			if !val.Installed && !downloaded {
-				if err = EnableItem(hub, &val); err != nil {
-					return fmt.Errorf("enabling '%s': %w", val.Name, err)
-				}
-			}
-
-			hubIdx[ptrtype][p] = val
-		}
-	}
-
-	err = DownloadItem(hub, target, overwrite)
-	if err != nil {
-		return fmt.Errorf("failed to download item: %w", err)
-	}
-
-	return nil
-}
-
-func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
-	tdir := hub.HubDir
-
-	// if user didn't --force, don't overwrite local, tainted, up-to-date files
-	if !overwrite {
-		if target.Tainted {
-			log.Debugf("%s : tainted, not updated", target.Name)
-			return nil
-		}
-
-		if target.UpToDate {
-			//  We still have to check if data files are present
-			log.Debugf("%s : up-to-date, not updated", target.Name)
-		}
-	}
-
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
-	}
-
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
-	}
-
-	h := sha256.New()
-	if _, err = h.Write(body); err != nil {
-		return fmt.Errorf("while hashing %s: %w", target.Name, err)
-	}
-
-	meow := fmt.Sprintf("%x", h.Sum(nil))
-	if meow != target.Versions[target.Version].Digest {
-		log.Errorf("Downloaded version doesn't match index, please 'hub update'")
-		log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
-
-		return fmt.Errorf("invalid download hash for %s", target.Name)
-	}
-
-	//all good, install
-	//check if parent dir exists
-	tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
-	parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
-
-	// ensure that target file is within target dir
-	finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
-	if err != nil {
-		return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
-	}
-
-	if !strings.HasPrefix(finalPath, tdir) {
-		return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
-	}
-
-	// check dir
-	if _, err = os.Stat(parentDir); os.IsNotExist(err) {
-		log.Debugf("%s doesn't exist, create", parentDir)
-
-		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
-			return fmt.Errorf("while creating parent directories: %w", err)
-		}
-	}
-
-	// check actual file
-	if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
-		log.Warningf("%s : overwrite", target.Name)
-		log.Debugf("target: %s/%s", tdir, target.RemotePath)
-	} else {
-		log.Infof("%s : OK", target.Name)
-	}
-
-	f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-	if err != nil {
-		return fmt.Errorf("while opening file: %w", err)
-	}
-
-	defer f.Close()
-
-	_, err = f.Write(body)
-	if err != nil {
-		return fmt.Errorf("while writing file: %w", err)
-	}
-
-	target.Downloaded = true
-	target.Tainted = false
-	target.UpToDate = true
-
-	if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
-		return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
-	}
-
-	hubIdx[target.Type][target.Name] = *target
-
-	return nil
-}
-
-func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
-	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
-
-	itemFile, err := os.Open(itemFilePath)
-	if err != nil {
-		return fmt.Errorf("while opening %s: %w", itemFilePath, err)
-	}
-
-	defer itemFile.Close()
-
-	if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil {
-		return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
-	}
-
-	return nil
-}
-
-func downloadData(dataFolder string, force bool, reader io.Reader) error {
-	var err error
-
-	dec := yaml.NewDecoder(reader)
-
-	for {
-		data := &DataSet{}
-
-		err = dec.Decode(data)
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-
-			return fmt.Errorf("while reading file: %w", err)
-		}
-
-		download := false
-
-		for _, dataS := range data.Data {
-			if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
-				download = true
-			}
-		}
-
-		if download || force {
-			err = GetData(data.Data, dataFolder)
-			if err != nil {
-				return fmt.Errorf("while getting data: %w", err)
-			}
-		}
-	}
-
-	return nil
-}

pkg/cwhub/download_test.go

@@ -1,52 +0,0 @@
-package cwhub
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-)
-
-func TestDownloadHubIdx(t *testing.T) {
-	back := RawFileURLTemplate
-	// bad url template
-	fmt.Println("Test 'bad URL'")
-
-	RawFileURLTemplate = "x"
-
-	ret, err := DownloadHubIdx(&csconfig.Hub{})
-	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
-		log.Errorf("unexpected error %s", err)
-	}
-
-	fmt.Printf("->%+v", ret)
-
-	// bad domain
-	fmt.Println("Test 'bad domain'")
-
-	RawFileURLTemplate = "https://baddomain/%s/%s"
-
-	ret, err = DownloadHubIdx(&csconfig.Hub{})
-	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
-		log.Errorf("unexpected error %s", err)
-	}
-
-	fmt.Printf("->%+v", ret)
-
-	// bad target path
-	fmt.Println("Test 'bad target path'")
-
-	RawFileURLTemplate = back
-
-	ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
-	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
-		log.Errorf("unexpected error %s", err)
-	}
-
-	RawFileURLTemplate = back
-
-	fmt.Printf("->%+v", ret)
-}

pkg/cwhub/enable.go

@@ -0,0 +1,190 @@
+package cwhub
+
+// Enable/disable items already downloaded
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
+// (eg. /etc/crowdsec/collections/xyz.yaml).
+// Raises an error if the path goes outside of the install dir.
+func (i *Item) installPath() (string, error) {
+	p := i.Type
+	if i.Stage != "" {
+		p = filepath.Join(p, i.Stage)
+	}
+
+	return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
+}
+
+// downloadPath returns the location of the actual config file in the hub
+// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
+// Raises an error if the path goes outside of the hub dir.
+func (i *Item) downloadPath() (string, error) {
+	ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
+	if err != nil {
+		return "", err
+	}
+
+	return ret, nil
+}
+
+// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
+func (i *Item) createInstallLink() error {
+	dest, err := i.installPath()
+	if err != nil {
+		return err
+	}
+
+	destDir := filepath.Dir(dest)
+	if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
+		return fmt.Errorf("while creating %s: %w", destDir, err)
+	}
+
+	if _, err = os.Lstat(dest); !os.IsNotExist(err) {
+		log.Infof("%s already exists.", dest)
+		return nil
+	}
+
+	src, err := i.downloadPath()
+	if err != nil {
+		return err
+	}
+
+	if err = os.Symlink(src, dest); err != nil {
+		return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
+	}
+
+	return nil
+}
+
+// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
+func (i *Item) enable() error {
+	if i.State.Installed {
+		if i.State.Tainted {
+			return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
+		}
+
+		if i.IsLocal() {
+			return fmt.Errorf("%s is local, won't enable", i.Name)
+		}
+
+		// if it's a collection, check sub-items even if the collection file itself is up-to-date
+		if i.State.UpToDate && !i.HasSubItems() {
+			log.Tracef("%s is installed and up-to-date, skip.", i.Name)
+			return nil
+		}
+	}
+
+	for _, sub := range i.SubItems() {
+		if err := sub.enable(); err != nil {
+			return fmt.Errorf("while installing %s: %w", sub.Name, err)
+		}
+	}
+
+	if err := i.createInstallLink(); err != nil {
+		return err
+	}
+
+	log.Infof("Enabled %s: %s", i.Type, i.Name)
+	i.State.Installed = true
+
+	return nil
+}
+
+// purge removes the actual config file that was downloaded.
+func (i *Item) purge() error {
+	if !i.State.Downloaded {
+		log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
+		return nil
+	}
+
+	src, err := i.downloadPath()
+	if err != nil {
+		return err
+	}
+
+	if err := os.Remove(src); err != nil {
+		if os.IsNotExist(err) {
+			log.Debugf("%s doesn't exist, no need to remove", src)
+			return nil
+		}
+
+		return fmt.Errorf("while removing file: %w", err)
+	}
+
+	i.State.Downloaded = false
+	log.Infof("Removed source file [%s]: %s", i.Name, src)
+
+	return nil
+}
+
+// removeInstallLink removes the symlink to the downloaded content.
+func (i *Item) removeInstallLink() error {
+	syml, err := i.installPath()
+	if err != nil {
+		return err
+	}
+
+	stat, err := os.Lstat(syml)
+	if err != nil {
+		return err
+	}
+
+	// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
+	if stat.Mode()&os.ModeSymlink == 0 {
+		log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
+		return fmt.Errorf("%s isn't managed by hub", i.Name)
+	}
+
+	hubpath, err := os.Readlink(syml)
+	if err != nil {
+		return fmt.Errorf("while reading symlink: %w", err)
+	}
+
+	src, err := i.downloadPath()
+	if err != nil {
+		return err
+	}
+
+	if hubpath != src {
+		log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
+		return fmt.Errorf("%s isn't managed by hub", i.Name)
+	}
+
+	if err := os.Remove(syml); err != nil {
+		return fmt.Errorf("while removing symlink: %w", err)
+	}
+
+	log.Infof("Removed symlink [%s]: %s", i.Name, syml)
+
+	return nil
+}
+
+// disable removes the install link, and optionally the downloaded content.
+func (i *Item) disable(purge bool, force bool) error {
+	err := i.removeInstallLink()
+	if os.IsNotExist(err) {
+		if !purge && !force {
+			link, _ := i.installPath()
+			return fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
+		}
+	} else if err != nil {
+		return err
+	}
+
+	i.State.Installed = false
+
+	if purge {
+		if err := i.purge(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/cwhub/enable_test.go

@@ -0,0 +1,141 @@
+package cwhub
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testInstall(hub *Hub, t *testing.T, item *Item) {
+	// Install the parser
+	_, err := item.downloadLatest(false, false)
+	require.NoError(t, err, "failed to download %s", item.Name)
+
+	err = hub.localSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
+
+	err = item.enable()
+	require.NoError(t, err, "failed to enable %s", item.Name)
+
+	err = hub.localSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
+}
+
+func testTaint(hub *Hub, t *testing.T, item *Item) {
+	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
+
+	// truncate the file
+	f, err := os.Create(item.State.LocalPath)
+	require.NoError(t, err)
+	f.Close()
+
+	// Local sync and check status
+	err = hub.localSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name)
+}
+
+func testUpdate(hub *Hub, t *testing.T, item *Item) {
+	assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name)
+
+	// Update it + check status
+	_, err := item.downloadLatest(true, true)
+	require.NoError(t, err, "failed to update %s", item.Name)
+
+	// Local sync and check status
+	err = hub.localSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
+}
+
+func testDisable(hub *Hub, t *testing.T, item *Item) {
+	assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
+
+	// Remove
+	err := item.disable(false, false)
+	require.NoError(t, err, "failed to disable %s", item.Name)
+
+	// Local sync and check status
+	err = hub.localSync()
+	require.NoError(t, err, "failed to run localSync")
+	require.Empty(t, hub.Warnings)
+
+	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
+	assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name)
+
+	// Purge
+	err = item.disable(true, false)
+	require.NoError(t, err, "failed to purge %s", item.Name)
+
+	// Local sync and check status
+	err = hub.localSync()
+	require.NoError(t, err, "failed to run localSync")
+	require.Empty(t, hub.Warnings)
+
+	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name)
+}
+
+func TestInstallParser(t *testing.T) {
+	/*
+	 - install a random parser
+	 - check its status
+	 - taint it
+	 - check its status
+	 - force update it
+	 - check its status
+	 - remove it
+	*/
+	hub := envSetup(t)
+
+	// map iteration is random by itself
+	for _, it := range hub.Items[PARSERS] {
+		testInstall(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testTaint(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testUpdate(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testDisable(hub, t, it)
+
+		break
+	}
+}
+
+func TestInstallCollection(t *testing.T) {
+	/*
+	 - install a random parser
+	 - check its status
+	 - taint it
+	 - check its status
+	 - force update it
+	 - check its status
+	 - remove it
+	*/
+	hub := envSetup(t)
+
+	// map iteration is random by itself
+	for _, it := range hub.Items[COLLECTIONS] {
+		testInstall(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testTaint(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testUpdate(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testDisable(hub, t, it)
+
+		break
+	}
+}

pkg/cwhub/errors.go

@@ -0,0 +1,21 @@
+package cwhub
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
+	ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
+)
+
+// IndexNotFoundError is returned when the remote hub index is not found.
+type IndexNotFoundError struct {
+	URL    string
+	Branch string
+}
+
+func (e IndexNotFoundError) Error() string {
+	return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch)
+}

pkg/cwhub/helpers.go

@@ -1,222 +1,373 @@
 package cwhub
 
+// Install, upgrade and remove items from the hub to the local configuration
+
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
 	"path/filepath"
 
 	"github.com/enescakir/emoji"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/mod/semver"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"slices"
 )
 
-// pick a hub branch corresponding to the current crowdsec version.
-func chooseHubBranch() string {
-	latest, err := cwversion.Latest()
-	if err != nil {
-		log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
-		//lint:ignore nilerr
-		return "master"
-	}
+// Install installs the item from the hub, downloading it if needed.
+func (i *Item) Install(force bool, downloadOnly bool) error {
+	if downloadOnly && i.State.Downloaded && i.State.UpToDate {
+		log.Infof("%s is already downloaded and up-to-date", i.Name)
 
-	csVersion := cwversion.VersionStrip()
-	if csVersion == latest {
-		log.Debugf("current version is equal to latest (%s)", csVersion)
-		return "master"
+		if !force {
+			return nil
+		}
 	}
 
-	// if current version is greater than the latest we are in pre-release
-	if semver.Compare(csVersion, latest) == 1 {
-		log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
-		return "master"
+	filePath, err := i.downloadLatest(force, true)
+	if err != nil {
+		return fmt.Errorf("while downloading %s: %w", i.Name, err)
 	}
 
-	if csVersion == "" {
-		log.Warning("Crowdsec version is not set, using master branch for the hub")
-		return "master"
+	if downloadOnly {
+		log.Infof("Downloaded %s to %s", i.Name, filePath)
+		return nil
 	}
 
-	log.Warnf("Crowdsec is not the latest version. "+
-		"Current version is '%s' and the latest stable version is '%s'. Please update it!",
-		csVersion, latest)
+	if err := i.enable(); err != nil {
+		return fmt.Errorf("while enabling %s: %w", i.Name, err)
+	}
 
-	log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
-		"added to Crowdsec Hub after CrowdSec %s", latest)
+	log.Infof("Enabled %s", i.Name)
 
-	return csVersion
+	return nil
 }
 
-// SetHubBranch sets the package variable that points to the hub branch.
-func SetHubBranch() {
-	// a branch is already set, or specified from the flags
-	if HubBranch != "" {
-		return
-	}
+// descendants returns a list of all (direct or indirect) dependencies of the item.
+func (i *Item) descendants() ([]*Item, error) {
+	var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
 
-	// use the branch corresponding to the crowdsec version
-	HubBranch = chooseHubBranch()
+	collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
+		if item == nil {
+			return nil
+		}
 
-	log.Debugf("Using branch '%s' for the hub", HubBranch)
-}
+		if visited[item] {
+			return nil
+		}
 
-func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
-	item := GetItem(obtype, name)
-	if item == nil {
-		return fmt.Errorf("unable to retrieve item: %s", name)
-	}
+		visited[item] = true
+
+		for _, subItem := range item.SubItems() {
+			if subItem == i {
+				return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
+			}
 
-	if downloadOnly && item.Downloaded && item.UpToDate {
-		log.Warningf("%s is already downloaded and up-to-date", item.Name)
+			*result = append(*result, subItem)
 
-		if !force {
-			return nil
+			err := collectSubItems(subItem, visited, result)
+			if err != nil {
+				return err
+			}
 		}
+
+		return nil
 	}
 
-	err := DownloadLatest(csConfig.Hub, item, force, true)
+	ret := []*Item{}
+	visited := map[*Item]bool{}
+
+	err := collectSubItems(i, visited, &ret)
 	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", item.Name, err)
+		return nil, err
 	}
 
-	if err = AddItem(obtype, *item); err != nil {
-		return fmt.Errorf("while adding %s: %w", item.Name, err)
-	}
+	return ret, nil
+}
 
-	if downloadOnly {
-		log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath))
-		return nil
+// Remove disables the item, optionally removing the downloaded content.
+func (i *Item) Remove(purge bool, force bool) (bool, error) {
+	if i.IsLocal() {
+		return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
 	}
 
-	err = EnableItem(csConfig.Hub, item)
-	if err != nil {
-		return fmt.Errorf("while enabling %s: %w", item.Name, err)
+	if i.State.Tainted && !force {
+		return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
 	}
 
-	if err := AddItem(obtype, *item); err != nil {
-		return fmt.Errorf("while adding %s: %w", item.Name, err)
+	if !i.State.Installed && !purge {
+		log.Infof("removing %s: not installed -- no need to remove", i.Name)
+		return false, nil
 	}
 
-	log.Infof("Enabled %s", item.Name)
+	removed := false
 
-	return nil
-}
+	descendants, err := i.descendants()
+	if err != nil {
+		return false, err
+	}
 
-// XXX this must return errors instead of log.Fatal
-func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) {
-	if name != "" {
-		item := GetItem(itemType, name)
-		if item == nil {
-			log.Fatalf("unable to retrieve: %s", name)
+	ancestors := i.Ancestors()
+
+	for _, sub := range i.SubItems() {
+		if !sub.State.Installed {
+			continue
 		}
 
-		err := DisableItem(csConfig.Hub, item, purge, forceAction)
+		// if the sub depends on a collection that is not a direct or indirect dependency
+		// of the current item, it is not removed
+		for _, subParent := range sub.Ancestors() {
+			if !purge && !subParent.State.Installed {
+				continue
+			}
 
-		if err != nil {
-			log.Fatalf("unable to disable %s : %v", item.Name, err)
+			// the ancestor that would block the removal of the sub item is also an ancestor
+			// of the item we are removing, so we don't want false warnings
+			// (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux,
+			// while we are removing crowdsecurity/sshd)
+			if slices.Contains(ancestors, subParent) {
+				continue
+			}
+
+			// the sub-item belongs to the item we are removing, but we already knew that
+			if subParent == i {
+				continue
+			}
+
+			if !slices.Contains(descendants, subParent) {
+				log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
+				continue
+			}
 		}
 
-		if err = AddItem(itemType, *item); err != nil {
-			log.Fatalf("unable to add %s: %v", item.Name, err)
+		subRemoved, err := sub.Remove(purge, force)
+		if err != nil {
+			return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
 		}
 
-		return
+		removed = removed || subRemoved
 	}
 
-	if !all {
-		log.Fatal("removing item: no item specified")
+	if err = i.disable(purge, force); err != nil {
+		return false, fmt.Errorf("while removing %s: %w", i.Name, err)
 	}
 
-	disabled := 0
+	removed = true
 
-	// remove all
-	for _, v := range GetItemMap(itemType) {
-		if !v.Installed {
-			continue
+	return removed, nil
+}
+
+// Upgrade downloads and applies the last version of the item from the hub.
+func (i *Item) Upgrade(force bool) (bool, error) {
+	updated := false
+
+	if !i.State.Downloaded {
+		return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
+	}
+
+	if !i.State.Installed {
+		return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name)
+	}
+
+	if i.State.UpToDate {
+		log.Infof("%s: up-to-date", i.Name)
+
+		if err := i.DownloadDataIfNeeded(force); err != nil {
+			return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
 		}
 
-		err := DisableItem(csConfig.Hub, &v, purge, forceAction)
-		if err != nil {
-			log.Fatalf("unable to disable %s : %v", v.Name, err)
+		if !force {
+			// no upgrade needed
+			return false, nil
 		}
+	}
+
+	if _, err := i.downloadLatest(force, true); err != nil {
+		return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
+	}
 
-		if err := AddItem(itemType, v); err != nil {
-			log.Fatalf("unable to add %s: %v", v.Name, err)
+	if !i.State.UpToDate {
+		if i.State.Tainted {
+			log.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
+		} else if i.IsLocal() {
+			log.Infof("%v %s is local", emoji.Prohibited, i.Name)
 		}
-		disabled++
+	} else {
+		// a check on stdout is used while scripting to know if the hub has been upgraded
+		// and a configuration reload is required
+		// TODO: use a better way to communicate this
+		fmt.Printf("updated %s\n", i.Name)
+		log.Infof("%v %s: updated", emoji.Package, i.Name)
+		updated = true
 	}
 
-	log.Infof("Disabled %d items", disabled)
+	return updated, nil
 }
 
-func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) {
-	updated := 0
-	found := false
+// downloadLatest downloads the latest version of the item to the hub directory.
+func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
+	log.Debugf("Downloading %s %s", i.Type, i.Name)
 
-	for _, v := range GetItemMap(itemType) {
-		if name != "" && name != v.Name {
+	for _, sub := range i.SubItems() {
+		if !sub.State.Installed && updateOnly && sub.State.Downloaded {
+			log.Debugf("skipping upgrade of %s: not installed", i.Name)
 			continue
 		}
 
-		if !v.Installed {
-			log.Tracef("skip %s, not installed", v.Name)
-			continue
-		}
+		log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly)
 
-		if !v.Downloaded {
-			log.Warningf("%s : not downloaded, please install.", v.Name)
-			continue
-		}
+		// recurse as it's a collection
+		if sub.HasSubItems() {
+			log.Tracef("collection, recurse")
 
-		found = true
+			if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil {
+				return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
+			}
+		}
 
-		if v.UpToDate {
-			log.Infof("%s : up-to-date", v.Name)
+		downloaded := sub.State.Downloaded
 
-			if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil {
-				log.Fatalf("%s : download failed : %v", v.Name, err)
-			}
+		if _, err := sub.download(overwrite); err != nil {
+			return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
+		}
 
-			if !force {
-				continue
+		// We need to enable an item when it has been added to a collection since latest release of the collection.
+		// We check if sub.Downloaded is false because maybe the item has been disabled by the user.
+		if !sub.State.Installed && !downloaded {
+			if err := sub.enable(); err != nil {
+				return "", fmt.Errorf("enabling '%s': %w", sub.Name, err)
 			}
 		}
+	}
 
-		if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil {
-			log.Fatalf("%s : download failed : %v", v.Name, err)
-		}
+	if !i.State.Installed && updateOnly && i.State.Downloaded {
+		log.Debugf("skipping upgrade of %s: not installed", i.Name)
+		return "", nil
+	}
 
-		if !v.UpToDate {
-			if v.Tainted {
-				log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
-			} else if v.Local {
-				log.Infof("%v %s is local", emoji.Prohibited, v.Name)
-			}
-		} else {
-			// this is used while scripting to know if the hub has been upgraded
-			// and a configuration reload is required
-			fmt.Printf("updated %s\n", v.Name)
-			log.Infof("%v %s : updated", emoji.Package, v.Name)
-			updated++
+	ret, err := i.download(overwrite)
+	if err != nil {
+		return "", fmt.Errorf("failed to download item: %w", err)
+	}
+
+	return ret, nil
+}
+
+// fetch downloads the item from the hub, verifies the hash and returns the content.
+func (i *Item) fetch() ([]byte, error) {
+	url, err := i.hub.remote.urlTo(i.RemotePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build hub item request: %w", err)
+	}
+
+	resp, err := hubClient.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("while downloading %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("while downloading %s: %w", url, err)
+	}
+
+	hash := sha256.New()
+	if _, err = hash.Write(body); err != nil {
+		return nil, fmt.Errorf("while hashing %s: %w", i.Name, err)
+	}
+
+	meow := hex.EncodeToString(hash.Sum(nil))
+	if meow != i.Versions[i.Version].Digest {
+		log.Errorf("Downloaded version doesn't match index, please 'hub update'")
+		log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
+
+		return nil, fmt.Errorf("invalid download hash for %s", i.Name)
+	}
+
+	return body, nil
+}
+
+// download downloads the item from the hub and writes it to the hub directory.
+func (i *Item) download(overwrite bool) (string, error) {
+	// if user didn't --force, don't overwrite local, tainted, up-to-date files
+	if !overwrite {
+		if i.State.Tainted {
+			log.Debugf("%s: tainted, not updated", i.Name)
+			return "", nil
 		}
 
-		if err := AddItem(itemType, v); err != nil {
-			log.Fatalf("unable to add %s: %v", v.Name, err)
+		if i.State.UpToDate {
+			//  We still have to check if data files are present
+			log.Debugf("%s: up-to-date, not updated", i.Name)
 		}
 	}
 
-	if !found && name == "" {
-		log.Infof("No %s installed, nothing to upgrade", itemType)
-	} else if !found {
-		log.Errorf("Item '%s' not found in hub", name)
-	} else if updated == 0 && found {
-		if name == "" {
-			log.Infof("All %s are already up-to-date", itemType)
-		} else {
-			log.Infof("Item '%s' is up-to-date", name)
-		}
-	} else if updated != 0 {
-		log.Infof("Upgraded %d items", updated)
+	body, err := i.fetch()
+	if err != nil {
+		return "", err
+	}
+
+	// all good, install
+
+	// ensure that target file is within target dir
+	finalPath, err := i.downloadPath()
+	if err != nil {
+		return "", err
+	}
+
+	parentDir := filepath.Dir(finalPath)
+
+	if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
+		return "", fmt.Errorf("while creating %s: %w", parentDir, err)
+	}
+
+	// check actual file
+	if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
+		log.Warningf("%s: overwrite", i.Name)
+		log.Debugf("target: %s", finalPath)
+	} else {
+		log.Infof("%s: OK", i.Name)
+	}
+
+	if err = os.WriteFile(finalPath, body, 0o644); err != nil {
+		return "", fmt.Errorf("while writing %s: %w", finalPath, err)
 	}
+
+	i.State.Downloaded = true
+	i.State.Tainted = false
+	i.State.UpToDate = true
+
+	if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
+		return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
+	}
+
+	return finalPath, nil
+}
+
+// DownloadDataIfNeeded downloads the data set for the item.
+func (i *Item) DownloadDataIfNeeded(force bool) error {
+	itemFilePath, err := i.installPath()
+	if err != nil {
+		return err
+	}
+
+	itemFile, err := os.Open(itemFilePath)
+	if err != nil {
+		return fmt.Errorf("while opening %s: %w", itemFilePath, err)
+	}
+
+	defer itemFile.Close()
+
+	if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile); err != nil {
+		return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
+	}
+
+	return nil
 }

pkg/cwhub/helpers_test.go

@@ -4,157 +4,187 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
-// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
-// We expect the new scenario to be installed
-func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection.
+// We expect the new scenario to be installed.
+func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
+	hub := envSetup(t)
 
 	// fresh install of collection
-	getHubIdxOrFail(t)
-
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
 
-	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
+	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+	require.NoError(t, item.Install(false, false))
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
 
 	// This is the scenario that gets added in next version of collection
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
 
-	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
+	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	pushUpdateToCollectionInHub()
 
-	if err := UpdateHubIdx(cfg.Hub); err != nil {
-		t.Fatalf("failed to download index : %s", err)
+	remote := &RemoteHubCfg{
+		URLTemplate: mockURLTemplate,
+		Branch:      "master",
+		IndexPath:   ".index.json",
 	}
 
-	getHubIdxOrFail(t)
+	hub, err := NewHub(hub.local, remote, true)
+	require.NoError(t, err, "failed to download index: %s", err)
+
+	hub = getHubOrFail(t, hub.local, remote)
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
 
-	UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
-	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
+	item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+	didUpdate, err := item.Upgrade(false)
+	require.NoError(t, err)
+	require.True(t, didUpdate)
+	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
 }
 
 // Install a collection, disable a scenario.
 // Upgrade should install should not enable/download the disabled scenario.
-func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
+	hub := envSetup(t)
 
 	// fresh install of collection
-	getHubIdxOrFail(t)
-
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-
-	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
-
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
-
-	RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
-	getHubIdxOrFail(t)
-	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-
-	if err := UpdateHubIdx(cfg.Hub); err != nil {
-		t.Fatalf("failed to download index : %s", err)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+
+	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+	require.NoError(t, item.Install(false, false))
+
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
+
+	item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
+	didRemove, err := item.Remove(false, false)
+	require.NoError(t, err)
+	require.True(t, didRemove)
+
+	remote := &RemoteHubCfg{
+		URLTemplate: mockURLTemplate,
+		Branch:      "master",
+		IndexPath:   ".index.json",
 	}
 
-	UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
-
-	getHubIdxOrFail(t)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	hub = getHubOrFail(t, hub.local, remote)
+	// scenario referenced by collection  was deleted hence, collection should be tainted
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+
+	hub, err = NewHub(hub.local, remote, true)
+	require.NoError(t, err, "failed to download index: %s", err)
+
+	item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+	didUpdate, err := item.Upgrade(false)
+	require.NoError(t, err)
+	require.False(t, didUpdate)
+
+	hub = getHubOrFail(t, hub.local, remote)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
 }
 
-func getHubIdxOrFail(t *testing.T) {
-	if err := GetHubIdx(getTestCfg().Hub); err != nil {
-		t.Fatalf("failed to load hub index")
-	}
+// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
+func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
+	hub, err := NewHub(local, remote, false)
+	require.NoError(t, err, "failed to load hub index")
+
+	return hub
 }
 
 // Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
 // Upgrade should not enable/download the disabled scenario.
 // Upgrade should install and enable the newly added scenario.
-func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
+	hub := envSetup(t)
 
 	// fresh install of collection
-	getHubIdxOrFail(t)
-
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-
-	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
-
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+
+	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+	require.NoError(t, item.Install(false, false))
+
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
+
+	item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
+	didRemove, err := item.Remove(false, false)
+	require.NoError(t, err)
+	require.True(t, didRemove)
+
+	remote := &RemoteHubCfg{
+		URLTemplate: mockURLTemplate,
+		Branch:      "master",
+		IndexPath:   ".index.json",
+	}
 
-	RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
-	getHubIdxOrFail(t)
+	hub = getHubOrFail(t, hub.local, remote)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
 	// we just removed. Nor should it install the newly added scenario
 	pushUpdateToCollectionInHub()
 
-	if err := UpdateHubIdx(cfg.Hub); err != nil {
-		t.Fatalf("failed to download index : %s", err)
-	}
+	hub, err = NewHub(hub.local, remote, true)
+	require.NoError(t, err, "failed to download index: %s", err)
+
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	hub = getHubOrFail(t, hub.local, remote)
 
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	getHubIdxOrFail(t)
+	item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+	didUpdate, err := item.Upgrade(false)
+	require.NoError(t, err)
+	require.True(t, didUpdate)
 
-	UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
-	getHubIdxOrFail(t)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	hub = getHubOrFail(t, hub.local, remote)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
 }
 
-func assertCollectionDepsInstalled(t *testing.T, collection string) {
+func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
 	t.Helper()
 
-	c := hubIdx[COLLECTIONS][collection]
-	require.NoError(t, CollecDepsCheck(&c))
+	c := hub.Items[COLLECTIONS][collection]
+	require.NoError(t, c.checkSubItemVersions())
 }
 
 func pushUpdateToCollectionInHub() {

pkg/cwhub/hub.go

@@ -0,0 +1,161 @@
+package cwhub
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+)
+
+// Hub is the main structure for the package.
+type Hub struct {
+	Items    HubItems // Items read from HubDir and InstallDir
+	local    *csconfig.LocalHubCfg
+	remote   *RemoteHubCfg
+	Warnings []string // Warnings encountered during sync
+}
+
+// GetDataDir returns the data directory, where data sets are installed.
+func (h *Hub) GetDataDir() string {
+	return h.local.InstallDataDir
+}
+
+// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
+// If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
+// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
+func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
+	if local == nil {
+		return nil, fmt.Errorf("no hub configuration found")
+	}
+
+	hub := &Hub{
+		local:  local,
+		remote: remote,
+	}
+
+	if updateIndex {
+		if err := hub.updateIndex(); err != nil {
+			return nil, err
+		}
+	}
+
+	log.Debugf("loading hub idx %s", local.HubIndexFile)
+
+	if err := hub.parseIndex(); err != nil {
+		return nil, fmt.Errorf("failed to load index: %w", err)
+	}
+
+	if err := hub.localSync(); err != nil {
+		return nil, fmt.Errorf("failed to sync items: %w", err)
+	}
+
+	return hub, nil
+}
+
+// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections.
+func (h *Hub) parseIndex() error {
+	bidx, err := os.ReadFile(h.local.HubIndexFile)
+	if err != nil {
+		return fmt.Errorf("unable to read index file: %w", err)
+	}
+
+	if err := json.Unmarshal(bidx, &h.Items); err != nil {
+		return fmt.Errorf("failed to unmarshal index: %w", err)
+	}
+
+	log.Debugf("%d item types in hub index", len(ItemTypes))
+
+	// Iterate over the different types to complete the struct
+	for _, itemType := range ItemTypes {
+		log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
+
+		for name, item := range h.Items[itemType] {
+			item.hub = h
+			item.Name = name
+
+			// if the item has no (redundant) author, take it from the json key
+			if item.Author == "" && strings.Contains(name, "/") {
+				item.Author = strings.Split(name, "/")[0]
+			}
+
+			item.Type = itemType
+			item.FileName = path.Base(item.RemotePath)
+
+			item.logMissingSubItems()
+		}
+	}
+
+	return nil
+}
+
+// ItemStats returns total counts of the hub items, including local and tainted.
+func (h *Hub) ItemStats() []string {
+	loaded := ""
+	local := 0
+	tainted := 0
+
+	for _, itemType := range ItemTypes {
+		if len(h.Items[itemType]) == 0 {
+			continue
+		}
+
+		loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
+
+		for _, item := range h.Items[itemType] {
+			if item.IsLocal() {
+				local++
+			}
+
+			if item.State.Tainted {
+				tainted++
+			}
+		}
+	}
+
+	loaded = strings.Trim(loaded, ", ")
+	if loaded == "" {
+		loaded = "0 items"
+	}
+
+	ret := []string{
+		fmt.Sprintf("Loaded: %s", loaded),
+	}
+
+	if local > 0 || tainted > 0 {
+		ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", local, tainted))
+	}
+
+	return ret
+}
+
+// updateIndex downloads the latest version of the index and writes it to disk if it changed.
+func (h *Hub) updateIndex() error {
+	body, err := h.remote.fetchIndex()
+	if err != nil {
+		return err
+	}
+
+	oldContent, err := os.ReadFile(h.local.HubIndexFile)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			log.Warningf("failed to read hub index: %s", err)
+		}
+	} else if bytes.Equal(body, oldContent) {
+		log.Info("hub index is up to date")
+		return nil
+	}
+
+	if err = os.WriteFile(h.local.HubIndexFile, body, 0o644); err != nil {
+		return fmt.Errorf("failed to write hub index: %w", err)
+	}
+
+	log.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body))
+
+	return nil
+}

pkg/cwhub/hub_test.go

@@ -0,0 +1,77 @@
+package cwhub
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
+)
+
+func TestInitHubUpdate(t *testing.T) {
+	hub := envSetup(t)
+
+	remote := &RemoteHubCfg{
+		URLTemplate: mockURLTemplate,
+		Branch:      "master",
+		IndexPath:   ".index.json",
+	}
+
+	_, err := NewHub(hub.local, remote, true)
+	require.NoError(t, err)
+}
+
+func TestUpdateIndex(t *testing.T) {
+	// bad url template
+	fmt.Println("Test 'bad URL'")
+
+	tmpIndex, err := os.CreateTemp("", "index.json")
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		os.Remove(tmpIndex.Name())
+	})
+
+	hub := envSetup(t)
+
+	hub.remote = &RemoteHubCfg{
+		URLTemplate: "x",
+		Branch:      "",
+		IndexPath:   "",
+	}
+
+	hub.local.HubIndexFile = tmpIndex.Name()
+
+	err = hub.updateIndex()
+	cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'")
+
+	// bad domain
+	fmt.Println("Test 'bad domain'")
+
+	hub.remote = &RemoteHubCfg{
+		URLTemplate: "https://baddomain/%s/%s",
+		Branch:      "master",
+		IndexPath:   ".index.json",
+	}
+
+	err = hub.updateIndex()
+	require.NoError(t, err)
+	// XXX: this is not failing
+	//	cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
+
+	// bad target path
+	fmt.Println("Test 'bad target path'")
+
+	hub.remote = &RemoteHubCfg{
+		URLTemplate: mockURLTemplate,
+		Branch:      "master",
+		IndexPath:   ".index.json",
+	}
+
+	hub.local.HubIndexFile = "/does/not/exist/index.json"
+
+	err = hub.updateIndex()
+	cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:")
+}

pkg/cwhub/install.go

@@ -1,214 +0,0 @@
-package cwhub
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-)
-
-func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
-	itempath := hub.HubDir + "/" + target.RemotePath
-
-	// disable hub file
-	if err := os.Remove(itempath); err != nil {
-		return target, fmt.Errorf("while removing file: %w", err)
-	}
-
-	target.Downloaded = false
-	log.Infof("Removed source file [%s]: %s", target.Name, itempath)
-	hubIdx[target.Type][target.Name] = target
-
-	return target, nil
-}
-
-// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
-func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error {
-	var err error
-
-	// already disabled, noop unless purge
-	if !target.Installed {
-		if purge {
-			*target, err = purgeItem(hub, *target)
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}
-
-	if target.Local {
-		return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
-	}
-
-	if target.Tainted && !force {
-		return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
-	}
-
-	// for a COLLECTIONS, disable sub-items
-	if target.Type == COLLECTIONS {
-		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
-			ptrtype := ItemTypes[idx]
-			for _, p := range ptr {
-				if val, ok := hubIdx[ptrtype][p]; ok {
-					// check if the item doesn't belong to another collection before removing it
-					toRemove := true
-
-					for _, collection := range val.BelongsToCollections {
-						if collection != target.Name {
-							toRemove = false
-							break
-						}
-					}
-
-					if toRemove {
-						err = DisableItem(hub, &val, purge, force)
-						if err != nil {
-							return fmt.Errorf("while disabling %s: %w", p, err)
-						}
-					} else {
-						log.Infof("%s was not removed because it belongs to another collection", val.Name)
-					}
-				} else {
-					log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
-				}
-			}
-		}
-	}
-
-	syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
-	if err != nil {
-		return err
-	}
-
-	stat, err := os.Lstat(syml)
-	if os.IsNotExist(err) {
-		// we only accept to "delete" non existing items if it's a forced purge
-		if !purge && !force {
-			return fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml)
-		}
-	} else {
-		// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
-		if stat.Mode()&os.ModeSymlink == 0 {
-			log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml)
-			return fmt.Errorf("%s isn't managed by hub", target.Name)
-		}
-
-		hubpath, err := os.Readlink(syml)
-		if err != nil {
-			return fmt.Errorf("while reading symlink: %w", err)
-		}
-
-		absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
-		if err != nil {
-			return fmt.Errorf("while abs path: %w", err)
-		}
-
-		if hubpath != absPath {
-			log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath)
-			return fmt.Errorf("%s isn't managed by hub", target.Name)
-		}
-
-		// remove the symlink
-		if err = os.Remove(syml); err != nil {
-			return fmt.Errorf("while removing symlink: %w", err)
-		}
-
-		log.Infof("Removed symlink [%s] : %s", target.Name, syml)
-	}
-
-	target.Installed = false
-
-	if purge {
-		*target, err = purgeItem(hub, *target)
-		if err != nil {
-			return err
-		}
-	}
-
-	hubIdx[target.Type][target.Name] = *target
-
-	return nil
-}
-
-// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
-// Handles collections recursively
-func EnableItem(hub *csconfig.Hub, target *Item) error {
-	var err error
-
-	parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
-
-	// create directories if needed
-	if target.Installed {
-		if target.Tainted {
-			return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
-		}
-
-		if target.Local {
-			return fmt.Errorf("%s is local, won't enable", target.Name)
-		}
-
-		// if it's a collection, check sub-items even if the collection file itself is up-to-date
-		if target.UpToDate && target.Type != COLLECTIONS {
-			log.Tracef("%s is installed and up-to-date, skip.", target.Name)
-			return nil
-		}
-	}
-
-	if _, err = os.Stat(parentDir); os.IsNotExist(err) {
-		log.Infof("%s doesn't exist, create", parentDir)
-
-		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
-			return fmt.Errorf("while creating directory: %w", err)
-		}
-	}
-
-	// install sub-items if it's a collection
-	if target.Type == COLLECTIONS {
-		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
-			ptrtype := ItemTypes[idx]
-			for _, p := range ptr {
-				val, ok := hubIdx[ptrtype][p]
-				if !ok {
-					return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
-				}
-
-				err = EnableItem(hub, &val)
-				if err != nil {
-					return fmt.Errorf("while installing %s: %w", p, err)
-				}
-			}
-		}
-	}
-
-	// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
-	if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) {
-		log.Infof("%s already exists.", parentDir+"/"+target.FileName)
-		return nil
-	}
-
-	// hub.ConfigDir + target.RemotePath
-	srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
-	if err != nil {
-		return fmt.Errorf("while getting source path: %w", err)
-	}
-
-	dstPath, err := filepath.Abs(parentDir + "/" + target.FileName)
-	if err != nil {
-		return fmt.Errorf("while getting destination path: %w", err)
-	}
-
-	if err = os.Symlink(srcPath, dstPath); err != nil {
-		return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
-	}
-
-	log.Infof("Enabled %s : %s", target.Type, target.Name)
-	target.Installed = true
-	hubIdx[target.Type][target.Name] = *target
-
-	return nil
-}

pkg/cwhub/items.go

@@ -0,0 +1,383 @@
+package cwhub
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/Masterminds/semver/v3"
+	"github.com/enescakir/emoji"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	// managed item types.
+	COLLECTIONS   = "collections"
+	PARSERS       = "parsers"
+	POSTOVERFLOWS = "postoverflows"
+	SCENARIOS     = "scenarios"
+)
+
+const (
+	versionUpToDate        = iota // the latest version from index is installed
+	versionUpdateAvailable        // not installed, or lower than latest
+	versionUnknown                // local file with no version, or invalid version number
+	versionFuture                 // local version is higher latest, but is included in the index: should not happen
+)
+
+var (
+	// The order is important, as it is used to range over sub-items in collections.
+	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
+)
+
+type HubItems map[string]map[string]*Item
+
+// ItemVersion is used to detect the version of a given item
+// by comparing the hash of each version to the local file.
+// If the item does not match any known version, it is considered tainted (modified).
+type ItemVersion struct {
+	Digest     string `json:"digest,omitempty" yaml:"digest,omitempty"`
+	Deprecated bool   `json:"deprecated,omitempty" yaml:"deprecated,omitempty"`
+}
+
+// ItemState is used to keep the local state (i.e. at runtime) of an item.
+// This data is not stored in the index, but is displayed with "cscli ... inspect".
+type ItemState struct {
+	LocalPath            string   `json:"local_path,omitempty" yaml:"local_path,omitempty"`
+	LocalVersion         string   `json:"local_version,omitempty" yaml:"local_version,omitempty"`
+	LocalHash            string   `json:"local_hash,omitempty" yaml:"local_hash,omitempty"`
+	Installed            bool     `json:"installed"`
+	Downloaded           bool     `json:"downloaded"`
+	UpToDate             bool     `json:"up_to_date"`
+	Tainted              bool     `json:"tainted"`
+	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
+}
+
+// Item is created from an index file and enriched with local info.
+type Item struct {
+	hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
+
+	State ItemState `json:"-" yaml:"-"` // local state, not stored in the index
+
+	Type        string   `json:"type,omitempty" yaml:"type,omitempty"`           // one of the ItemTypes
+	Stage       string   `json:"stage,omitempty" yaml:"stage,omitempty"`         // Stage for parser|postoverflow: s00-raw/s01-...
+	Name        string   `json:"name,omitempty" yaml:"name,omitempty"`           // usually "author/name"
+	FileName    string   `json:"file_name,omitempty" yaml:"file_name,omitempty"` // eg. apache2-logs.yaml
+	Description string   `json:"description,omitempty" yaml:"description,omitempty"`
+	Author      string   `json:"author,omitempty" yaml:"author,omitempty"`
+	References  []string `json:"references,omitempty" yaml:"references,omitempty"`
+
+	RemotePath string                 `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
+	Version    string                 `json:"version,omitempty" yaml:"version,omitempty"`  // the last available version
+	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                // all the known versions
+
+	// if it's a collection, it can have sub items
+	Parsers       []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
+	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
+	Scenarios     []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
+	Collections   []string `json:"collections,omitempty" yaml:"collections,omitempty"`
+}
+
+// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
+func (i *Item) HasSubItems() bool {
+	return i.Type == COLLECTIONS
+}
+
+// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
+func (i *Item) IsLocal() bool {
+	return i.State.Installed && !i.State.Downloaded
+}
+
+// MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
+// It must not use a pointer receiver.
+func (i Item) MarshalJSON() ([]byte, error) {
+	type Alias Item
+
+	return json.Marshal(&struct {
+		Alias
+		// we have to repeat the fields here, json will have inline support in v2
+		LocalPath            string   `json:"local_path,omitempty"`
+		LocalVersion         string   `json:"local_version,omitempty"`
+		LocalHash            string   `json:"local_hash,omitempty"`
+		Installed            bool     `json:"installed"`
+		Downloaded           bool     `json:"downloaded"`
+		UpToDate             bool     `json:"up_to_date"`
+		Tainted              bool     `json:"tainted"`
+		Local                bool     `json:"local"`
+		BelongsToCollections []string `json:"belongs_to_collections,omitempty"`
+	}{
+		Alias:                Alias(i),
+		LocalPath:            i.State.LocalPath,
+		LocalVersion:         i.State.LocalVersion,
+		LocalHash:            i.State.LocalHash,
+		Installed:            i.State.Installed,
+		Downloaded:           i.State.Downloaded,
+		UpToDate:             i.State.UpToDate,
+		Tainted:              i.State.Tainted,
+		BelongsToCollections: i.State.BelongsToCollections,
+		Local:                i.IsLocal(),
+	})
+}
+
+// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw".
+// It must not use a pointer receiver.
+func (i Item) MarshalYAML() (interface{}, error) {
+	type Alias Item
+
+	return &struct {
+		Alias `yaml:",inline"`
+		State ItemState `yaml:",inline"`
+		Local bool      `yaml:"local"`
+	}{
+		Alias: Alias(i),
+		State: i.State,
+		Local: i.IsLocal(),
+	}, nil
+}
+
+// SubItems returns a slice of sub-items, excluding the ones that were not found.
+func (i *Item) SubItems() []*Item {
+	sub := make([]*Item, 0)
+
+	for _, name := range i.Parsers {
+		s := i.hub.GetItem(PARSERS, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
+	for _, name := range i.PostOverflows {
+		s := i.hub.GetItem(POSTOVERFLOWS, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
+	for _, name := range i.Scenarios {
+		s := i.hub.GetItem(SCENARIOS, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
+	for _, name := range i.Collections {
+		s := i.hub.GetItem(COLLECTIONS, name)
+		if s == nil {
+			continue
+		}
+
+		sub = append(sub, s)
+	}
+
+	return sub
+}
+
+func (i *Item) logMissingSubItems() {
+	if !i.HasSubItems() {
+		return
+	}
+
+	for _, subName := range i.Parsers {
+		if i.hub.GetItem(PARSERS, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name)
+		}
+	}
+
+	for _, subName := range i.Scenarios {
+		if i.hub.GetItem(SCENARIOS, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name)
+		}
+	}
+
+	for _, subName := range i.PostOverflows {
+		if i.hub.GetItem(POSTOVERFLOWS, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name)
+		}
+	}
+
+	for _, subName := range i.Collections {
+		if i.hub.GetItem(COLLECTIONS, subName) == nil {
+			log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name)
+		}
+	}
+}
+
+// Ancestors returns a slice of items (typically collections) that have this item as a direct or indirect dependency.
+func (i *Item) Ancestors() []*Item {
+	ret := make([]*Item, 0)
+
+	for _, parentName := range i.State.BelongsToCollections {
+		parent := i.hub.GetItem(COLLECTIONS, parentName)
+		if parent == nil {
+			continue
+		}
+
+		ret = append(ret, parent)
+	}
+
+	return ret
+}
+
+// InstallStatus returns the status of the item as a string and an emoji
+// (eg. "enabled,update-available" and emoji.Warning).
+func (i *Item) InstallStatus() (string, emoji.Emoji) {
+	status := "disabled"
+	ok := false
+
+	if i.State.Installed {
+		ok = true
+		status = "enabled"
+	}
+
+	managed := true
+	if i.IsLocal() {
+		managed = false
+		status += ",local"
+	}
+
+	warning := false
+	if i.State.Tainted {
+		warning = true
+		status += ",tainted"
+	} else if !i.State.UpToDate && !i.IsLocal() {
+		warning = true
+		status += ",update-available"
+	}
+
+	emo := emoji.QuestionMark
+
+	switch {
+	case !managed:
+		emo = emoji.House
+	case !i.State.Installed:
+		emo = emoji.Prohibited
+	case warning:
+		emo = emoji.Warning
+	case ok:
+		emo = emoji.CheckMark
+	}
+
+	return status, emo
+}
+
+// versionStatus returns the status of the item version compared to the hub version.
+// semver requires the 'v' prefix.
+func (i *Item) versionStatus() int {
+	local, err := semver.NewVersion(i.State.LocalVersion)
+	if err != nil {
+		return versionUnknown
+	}
+
+	// hub versions are already validated while syncing, ignore errors
+	latest, _ := semver.NewVersion(i.Version)
+
+	if local.LessThan(latest) {
+		return versionUpdateAvailable
+	}
+
+	if local.Equal(latest) {
+		return versionUpToDate
+	}
+
+	return versionFuture
+}
+
+// validPath returns true if the (relative) path is allowed for the item.
+// dirNname: the directory name (ie. crowdsecurity).
+// fileName: the filename (ie. apache2-logs.yaml).
+func (i *Item) validPath(dirName, fileName string) bool {
+	return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
+}
+
+// GetItemMap returns the map of items for a given type.
+func (h *Hub) GetItemMap(itemType string) map[string]*Item {
+	return h.Items[itemType]
+}
+
+// GetItem returns an item from hub based on its type and full name (author/name).
+func (h *Hub) GetItem(itemType string, itemName string) *Item {
+	return h.GetItemMap(itemType)[itemName]
+}
+
+// GetItemNames returns a slice of (full) item names for a given type
+// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
+func (h *Hub) GetItemNames(itemType string) []string {
+	m := h.GetItemMap(itemType)
+	if m == nil {
+		return nil
+	}
+
+	names := make([]string, 0, len(m))
+	for k := range m {
+		names = append(names, k)
+	}
+
+	return names
+}
+
+// GetAllItems returns a slice of all the items of a given type, installed or not.
+func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
+	items, ok := h.Items[itemType]
+	if !ok {
+		return nil, fmt.Errorf("no %s in the hub index", itemType)
+	}
+
+	ret := make([]*Item, len(items))
+
+	idx := 0
+
+	for _, item := range items {
+		ret[idx] = item
+		idx++
+	}
+
+	return ret, nil
+}
+
+// GetInstalledItems returns a slice of the installed items of a given type.
+func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
+	items, ok := h.Items[itemType]
+	if !ok {
+		return nil, fmt.Errorf("no %s in the hub index", itemType)
+	}
+
+	retItems := make([]*Item, 0)
+
+	for _, item := range items {
+		if item.State.Installed {
+			retItems = append(retItems, item)
+		}
+	}
+
+	return retItems, nil
+}
+
+// GetInstalledItemNames returns the names of the installed items of a given type.
+func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
+	items, err := h.GetInstalledItems(itemType)
+	if err != nil {
+		return nil, err
+	}
+
+	retStr := make([]string, len(items))
+
+	for idx, it := range items {
+		retStr[idx] = it.Name
+	}
+
+	return retStr, nil
+}
+
+// SortItemSlice sorts a slice of items by name, case insensitive.
+func SortItemSlice(items []*Item) {
+	sort.Slice(items, func(i, j int) bool {
+		return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
+	})
+}

pkg/cwhub/items_test.go

@@ -0,0 +1,71 @@
+package cwhub
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestItemStatus(t *testing.T) {
+	hub := envSetup(t)
+
+	// get existing map
+	x := hub.GetItemMap(COLLECTIONS)
+	require.NotEmpty(t, x)
+
+	// Get item: good and bad
+	for k := range x {
+		item := hub.GetItem(COLLECTIONS, k)
+		require.NotNil(t, item)
+
+		item.State.Installed = true
+		item.State.UpToDate = false
+		item.State.Tainted = false
+		item.State.Downloaded = true
+
+		txt, _ := item.InstallStatus()
+		require.Equal(t, "enabled,update-available", txt)
+
+		item.State.Installed = true
+		item.State.UpToDate = false
+		item.State.Tainted = false
+		item.State.Downloaded = false
+
+		txt, _ = item.InstallStatus()
+		require.Equal(t, "enabled,local", txt)
+	}
+
+	stats := hub.ItemStats()
+	require.Equal(t, []string{
+		"Loaded: 2 parsers, 1 scenarios, 3 collections",
+		"Unmanaged items: 3 local, 0 tainted",
+	}, stats)
+}
+
+func TestGetters(t *testing.T) {
+	hub := envSetup(t)
+
+	// get non existing map
+	empty := hub.GetItemMap("ratata")
+	require.Nil(t, empty)
+
+	// get existing map
+	x := hub.GetItemMap(COLLECTIONS)
+	require.NotEmpty(t, x)
+
+	// Get item: good and bad
+	for k := range x {
+		empty := hub.GetItem(COLLECTIONS, k+"nope")
+		require.Nil(t, empty)
+
+		item := hub.GetItem(COLLECTIONS, k)
+		require.NotNil(t, item)
+
+		// Add item and get it
+		item.Name += "nope"
+		hub.Items[item.Type][item.Name] = item
+
+		newitem := hub.GetItem(COLLECTIONS, item.Name)
+		require.NotNil(t, newitem)
+	}
+}

pkg/cwhub/leakybucket.go

@@ -0,0 +1,53 @@
+package cwhub
+
+// Resolve a symlink to find the hub item it points to.
+// This file is used only by pkg/leakybucket
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary.
+func itemKey(itemPath string) (string, error) {
+	f, err := os.Lstat(itemPath)
+	if err != nil {
+		return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
+	}
+
+	if f.Mode()&os.ModeSymlink == 0 {
+		// it's not a symlink, so the filename itsef should be the key
+		return filepath.Base(itemPath), nil
+	}
+
+	// resolve the symlink to hub file
+	pathInHub, err := os.Readlink(itemPath)
+	if err != nil {
+		return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
+	}
+
+	author := filepath.Base(filepath.Dir(pathInHub))
+
+	fname := filepath.Base(pathInHub)
+	fname = strings.TrimSuffix(fname, ".yaml")
+	fname = strings.TrimSuffix(fname, ".yml")
+
+	return fmt.Sprintf("%s/%s", author, fname), nil
+}
+
+// GetItemByPath retrieves an item from the hub index based on its local path.
+func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
+	itemKey, err := itemKey(itemPath)
+	if err != nil {
+		return nil, err
+	}
+
+	item := h.GetItem(itemType, itemKey)
+	if item == nil {
+		return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
+	}
+
+	return item, nil
+}

pkg/cwhub/loader.go

@@ -1,552 +0,0 @@
-package cwhub
-
-import (
-	"crypto/sha256"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-)
-
-func isYAMLFileName(path string) bool {
-	return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
-}
-
-func validItemFileName(vname string, fauthor string, fname string) bool {
-	return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml")
-}
-
-func handleSymlink(path string) (string, error) {
-	hubpath, err := os.Readlink(path)
-	if err != nil {
-		return "", fmt.Errorf("unable to read symlink of %s", path)
-	}
-	// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
-	_, err = os.Lstat(hubpath)
-	if os.IsNotExist(err) {
-		log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
-		// remove the symlink
-		if err = os.Remove(path); err != nil {
-			return "", fmt.Errorf("failed to unlink %s: %w", path, err)
-		}
-
-		// XXX: is this correct?
-		return "", nil
-	}
-
-	return hubpath, nil
-}
-
-func getSHA256(filepath string) (string, error) {
-	f, err := os.Open(filepath)
-	if err != nil {
-		return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
-	}
-
-	defer f.Close()
-
-	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
-		return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}
-
-type Walker struct {
-	// the walk/parserVisit function can't receive extra args
-	hubdir     string
-	installdir string
-}
-
-func NewWalker(hub *csconfig.Hub) Walker {
-	return Walker{
-		hubdir:     hub.HubDir,
-		installdir: hub.InstallDir,
-	}
-}
-
-type itemFileInfo struct {
-	fname   string
-	stage   string
-	ftype   string
-	fauthor string
-}
-
-func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
-	ret := itemFileInfo{}
-	inhub := false
-
-	subs := strings.Split(path, string(os.PathSeparator))
-
-	log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir)
-	log.Tracef("subs:%v", subs)
-	// we're in hub (~/.hub/hub/)
-	if strings.HasPrefix(path, w.hubdir) {
-		log.Tracef("in hub dir")
-
-		inhub = true
-		//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
-		//.../hub/scenarios/crowdsec/ssh_bf.yaml
-		//.../hub/profiles/crowdsec/linux.yaml
-		if len(subs) < 4 {
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
-		}
-
-		ret.fname = subs[len(subs)-1]
-		ret.fauthor = subs[len(subs)-2]
-		ret.stage = subs[len(subs)-3]
-		ret.ftype = subs[len(subs)-4]
-	} else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec/<type>/...
-		log.Tracef("in install dir")
-		if len(subs) < 3 {
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
-		}
-		///.../config/parser/stage/file.yaml
-		///.../config/postoverflow/stage/file.yaml
-		///.../config/scenarios/scenar.yaml
-		///.../config/collections/linux.yaml //file is empty
-		ret.fname = subs[len(subs)-1]
-		ret.stage = subs[len(subs)-2]
-		ret.ftype = subs[len(subs)-3]
-		ret.fauthor = ""
-	} else {
-		return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir)
-	}
-
-	log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
-	// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
-
-	if ret.stage == SCENARIOS {
-		ret.ftype = SCENARIOS
-		ret.stage = ""
-	} else if ret.stage == COLLECTIONS {
-		ret.ftype = COLLECTIONS
-		ret.stage = ""
-	} else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW {
-		// its a PARSER / PARSER_OVFLW with a stage
-		return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
-	}
-
-	log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
-
-	return ret, inhub, nil
-}
-
-func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
-	var (
-		local   bool
-		hubpath string
-	)
-
-	if err != nil {
-		log.Debugf("while syncing hub dir: %s", err)
-		// there is a path error, we ignore the file
-		return nil
-	}
-
-	path, err = filepath.Abs(path)
-	if err != nil {
-		return err
-	}
-
-	// we only care about files
-	if f == nil || f.IsDir() {
-		return nil
-	}
-
-	if !isYAMLFileName(f.Name()) {
-		return nil
-	}
-
-	info, inhub, err := w.getItemInfo(path)
-	if err != nil {
-		return err
-	}
-
-	/*
-		we can encounter 'collections' in the form of a symlink :
-		/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
-		when the collection is installed, both files are created
-	*/
-	// non symlinks are local user files or hub files
-	if f.Type()&os.ModeSymlink == 0 {
-		local = true
-
-		log.Tracef("%s isn't a symlink", path)
-	} else {
-		hubpath, err = handleSymlink(path)
-		if err != nil {
-			return err
-		}
-		log.Tracef("%s points to %s", path, hubpath)
-
-		if hubpath == "" {
-			// XXX: is this correct?
-			return nil
-		}
-	}
-
-	// if it's not a symlink and not in hub, it's a local file, don't bother
-	if local && !inhub {
-		log.Tracef("%s is a local file, skip", path)
-		skippedLocal++
-		//	log.Infof("local scenario, skip.")
-
-		_, fileName := filepath.Split(path)
-
-		hubIdx[info.ftype][info.fname] = Item{
-			Name:      info.fname,
-			Stage:     info.stage,
-			Installed: true,
-			Type:      info.ftype,
-			Local:     true,
-			LocalPath: path,
-			UpToDate:  true,
-			FileName:  fileName,
-		}
-
-		return nil
-	}
-
-	// try to find which configuration item it is
-	log.Tracef("check [%s] of %s", info.fname, info.ftype)
-
-	match := false
-
-	for name, item := range hubIdx[info.ftype] {
-		log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
-
-		if info.fname != item.FileName {
-			log.Tracef("%s != %s (filename)", info.fname, item.FileName)
-			continue
-		}
-
-		// wrong stage
-		if item.Stage != info.stage {
-			continue
-		}
-
-		// if we are walking hub dir, just mark present files as downloaded
-		if inhub {
-			// wrong author
-			if info.fauthor != item.Author {
-				continue
-			}
-
-			// wrong file
-			if !validItemFileName(item.Name, info.fauthor, info.fname) {
-				continue
-			}
-
-			if path == w.hubdir+"/"+item.RemotePath {
-				log.Tracef("marking %s as downloaded", item.Name)
-				item.Downloaded = true
-			}
-		} else if !hasPathSuffix(hubpath, item.RemotePath) {
-			// wrong file
-			// <type>/<stage>/<author>/<name>.yaml
-			continue
-		}
-
-		sha, err := getSHA256(path)
-		if err != nil {
-			log.Fatalf("Failed to get sha of %s : %v", path, err)
-		}
-
-		// let's reverse sort the versions to deal with hash collisions (#154)
-		versions := make([]string, 0, len(item.Versions))
-		for k := range item.Versions {
-			versions = append(versions, k)
-		}
-
-		sort.Sort(sort.Reverse(sort.StringSlice(versions)))
-
-		for _, version := range versions {
-			val := item.Versions[version]
-			if sha != val.Digest {
-				// log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
-				continue
-			}
-
-			// we got an exact match, update struct
-
-			item.Downloaded = true
-			item.LocalHash = sha
-
-			if !inhub {
-				log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version)
-				item.LocalPath = path
-				item.LocalVersion = version
-				item.Tainted = false
-				// if we're walking the hub, present file doesn't means installed file
-				item.Installed = true
-			}
-
-			if version == item.Version {
-				log.Tracef("%s is up-to-date", item.Name)
-				item.UpToDate = true
-			}
-
-			match = true
-
-			break
-		}
-
-		if !match {
-			log.Tracef("got tainted match for %s: %s", item.Name, path)
-
-			skippedTainted++
-			// the file and the stage is right, but the hash is wrong, it has been tainted by user
-			if !inhub {
-				item.LocalPath = path
-				item.Installed = true
-			}
-
-			item.UpToDate = false
-			item.LocalVersion = "?"
-			item.Tainted = true
-			item.LocalHash = sha
-		}
-
-		// update the entry if appropriate
-		// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
-		// 	fmt.Printf("Updating %s", k)
-		// 	hubIdx[ftype][k] = v
-		// } else if !inhub {
-
-		// } else if
-		hubIdx[info.ftype][name] = item
-
-		return nil
-	}
-
-	log.Infof("Ignoring file %s of type %s", path, info.ftype)
-
-	return nil
-}
-
-func CollecDepsCheck(v *Item) error {
-	if v.versionStatus() != 0 { // not up-to-date
-		log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
-		return nil
-	}
-
-	if v.Type != COLLECTIONS {
-		return nil
-	}
-
-	// if it's a collection, ensure all the items are installed, or tag it as tainted
-	log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
-
-	for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
-		sliceType := ItemTypes[idx]
-		for _, subName := range itemSlice {
-			subItem, ok := hubIdx[sliceType][subName]
-			if !ok {
-				log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name)
-			}
-
-			log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
-
-			if !v.Installed {
-				continue
-			}
-
-			if subItem.Type == COLLECTIONS {
-				log.Tracef("collec, recurse.")
-
-				if err := CollecDepsCheck(&subItem); err != nil {
-					if subItem.Tainted {
-						v.Tainted = true
-					}
-
-					return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
-				}
-
-				hubIdx[sliceType][subName] = subItem
-			}
-
-			// propagate the state of sub-items to set
-			if subItem.Tainted {
-				v.Tainted = true
-				return fmt.Errorf("tainted %s %s, tainted", sliceType, subName)
-			}
-
-			if !subItem.Installed && v.Installed {
-				v.Tainted = true
-				return fmt.Errorf("missing %s %s, tainted", sliceType, subName)
-			}
-
-			if !subItem.UpToDate {
-				v.UpToDate = false
-				return fmt.Errorf("outdated %s %s", sliceType, subName)
-			}
-
-			skip := false
-
-			for idx := range subItem.BelongsToCollections {
-				if subItem.BelongsToCollections[idx] == v.Name {
-					skip = true
-				}
-			}
-
-			if !skip {
-				subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
-			}
-
-			hubIdx[sliceType][subName] = subItem
-
-			log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
-		}
-	}
-
-	return nil
-}
-
-func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
-	warnings := []string{}
-
-	// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
-	for _, scan := range ItemTypes {
-		cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
-		if err != nil {
-			log.Errorf("failed %s : %s", cpath, err)
-		}
-
-		err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit)
-		if err != nil {
-			return warnings, err
-		}
-	}
-
-	for name, item := range hubIdx[COLLECTIONS] {
-		if !item.Installed {
-			continue
-		}
-
-		vs := item.versionStatus()
-		switch vs {
-		case 0: // latest
-			if err := CollecDepsCheck(&item); err != nil {
-				warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
-				hubIdx[COLLECTIONS][name] = item
-			}
-		case 1: // not up-to-date
-			warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
-		default: // version is higher than the highest available from hub?
-			warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
-		}
-
-		log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
-	}
-
-	return warnings, nil
-}
-
-// Updates the info from HubInit() with the local state
-func LocalSync(hub *csconfig.Hub) ([]string, error) {
-	skippedLocal = 0
-	skippedTainted = 0
-
-	warnings, err := SyncDir(hub, hub.InstallDir)
-	if err != nil {
-		return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err)
-	}
-
-	_, err = SyncDir(hub, hub.HubDir)
-	if err != nil {
-		return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err)
-	}
-
-	return warnings, nil
-}
-
-func GetHubIdx(hub *csconfig.Hub) error {
-	if hub == nil {
-		return fmt.Errorf("no configuration found for hub")
-	}
-
-	log.Debugf("loading hub idx %s", hub.HubIndexFile)
-
-	bidx, err := os.ReadFile(hub.HubIndexFile)
-	if err != nil {
-		return fmt.Errorf("unable to read index file: %w", err)
-	}
-
-	ret, err := LoadPkgIndex(bidx)
-	if err != nil {
-		if !errors.Is(err, ErrMissingReference) {
-			return fmt.Errorf("unable to load existing index: %w", err)
-		}
-
-		// XXX: why the error check if we bail out anyway?
-		return err
-	}
-
-	hubIdx = ret
-
-	_, err = LocalSync(hub)
-	if err != nil {
-		return fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
-	}
-
-	return nil
-}
-
-// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections
-func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
-	var (
-		RawIndex     map[string]map[string]Item
-		missingItems []string
-	)
-
-	if err := json.Unmarshal(buff, &RawIndex); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal index: %w", err)
-	}
-
-	log.Debugf("%d item types in hub index", len(ItemTypes))
-
-	// Iterate over the different types to complete the struct
-	for _, itemType := range ItemTypes {
-		log.Tracef("%d item", len(RawIndex[itemType]))
-
-		for name, item := range RawIndex[itemType] {
-			item.Name = name
-			item.Type = itemType
-			x := strings.Split(item.RemotePath, "/")
-			item.FileName = x[len(x)-1]
-			RawIndex[itemType][name] = item
-
-			if itemType != COLLECTIONS {
-				continue
-			}
-
-			// if it's a collection, check its sub-items are present
-			// XXX should be done later
-			for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
-				ptrtype := ItemTypes[idx]
-				for _, p := range ptr {
-					if _, ok := RawIndex[ptrtype][p]; !ok {
-						log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
-						missingItems = append(missingItems, p)
-					}
-				}
-			}
-		}
-	}
-
-	if len(missingItems) > 0 {
-		return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
-	}
-
-	return RawIndex, nil
-}

pkg/cwhub/remote.go

@@ -0,0 +1,61 @@
+package cwhub
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// RemoteHubCfg is used to retrieve index and items from the remote hub.
+type RemoteHubCfg struct {
+	Branch      string
+	URLTemplate string
+	IndexPath   string
+}
+
+// urlTo builds the URL to download a file from the remote hub.
+func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
+	if r == nil {
+		return "", ErrNilRemoteHub
+	}
+
+	// the template must contain two string placeholders
+	if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
+		return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
+	}
+
+	return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
+}
+
+// fetchIndex downloads the index from the hub and returns the content.
+func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
+	if r == nil {
+		return nil, ErrNilRemoteHub
+	}
+
+	url, err := r.urlTo(r.IndexPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build hub index request: %w", err)
+	}
+
+	resp, err := hubClient.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed http request for hub index: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusNotFound {
+			return nil, IndexNotFoundError{url, r.Branch}
+		}
+
+		return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
+	}
+
+	return body, nil
+}

pkg/cwhub/sync.go

@@ -0,0 +1,498 @@
+package cwhub
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/Masterminds/semver/v3"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+)
+
+func isYAMLFileName(path string) bool {
+	return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
+}
+
+// linkTarget returns the target of a symlink, or empty string if it's dangling.
+func linkTarget(path string) (string, error) {
+	hubpath, err := os.Readlink(path)
+	if err != nil {
+		return "", fmt.Errorf("unable to read symlink: %s", path)
+	}
+
+	log.Tracef("symlink %s -> %s", path, hubpath)
+
+	_, err = os.Lstat(hubpath)
+	if os.IsNotExist(err) {
+		log.Infof("link target does not exist: %s -> %s", path, hubpath)
+		return "", nil
+	}
+
+	return hubpath, nil
+}
+
+func getSHA256(filepath string) (string, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
+	}
+
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+// information used to create a new Item, from a file path.
+type itemFileInfo struct {
+	inhub   bool
+	fname   string
+	stage   string
+	ftype   string
+	fauthor string
+}
+
+func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
+	var ret *itemFileInfo
+
+	hubDir := h.local.HubDir
+	installDir := h.local.InstallDir
+
+	subs := strings.Split(path, string(os.PathSeparator))
+
+	log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
+	log.Tracef("subs:%v", subs)
+	// we're in hub (~/.hub/hub/)
+	if strings.HasPrefix(path, hubDir) {
+		log.Tracef("in hub dir")
+
+		//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
+		//.../hub/scenarios/crowdsec/ssh_bf.yaml
+		//.../hub/profiles/crowdsec/linux.yaml
+		if len(subs) < 4 {
+			return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
+		}
+
+		ret = &itemFileInfo{
+			inhub:   true,
+			fname:   subs[len(subs)-1],
+			fauthor: subs[len(subs)-2],
+			stage:   subs[len(subs)-3],
+			ftype:   subs[len(subs)-4],
+		}
+	} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
+		log.Tracef("in install dir")
+		if len(subs) < 3 {
+			return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
+		}
+		///.../config/parser/stage/file.yaml
+		///.../config/postoverflow/stage/file.yaml
+		///.../config/scenarios/scenar.yaml
+		///.../config/collections/linux.yaml //file is empty
+		ret = &itemFileInfo{
+			inhub:   false,
+			fname:   subs[len(subs)-1],
+			stage:   subs[len(subs)-2],
+			ftype:   subs[len(subs)-3],
+			fauthor: "",
+		}
+	} else {
+		return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
+	}
+
+	log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
+
+	if ret.stage == SCENARIOS {
+		ret.ftype = SCENARIOS
+		ret.stage = ""
+	} else if ret.stage == COLLECTIONS {
+		ret.ftype = COLLECTIONS
+		ret.stage = ""
+	} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
+		// it's a PARSER / POSTOVERFLOW with a stage
+		return nil, fmt.Errorf("unknown configuration type for file '%s'", path)
+	}
+
+	log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
+
+	return ret, nil
+}
+
+// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
+func sortedVersions(raw []string) ([]string, error) {
+	vs := make([]*semver.Version, len(raw))
+
+	for idx, r := range raw {
+		v, err := semver.NewVersion(r)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", r, err)
+		}
+
+		vs[idx] = v
+	}
+
+	sort.Sort(sort.Reverse(semver.Collection(vs)))
+
+	ret := make([]string, len(vs))
+	for idx, v := range vs {
+		ret[idx] = v.Original()
+	}
+
+	return ret, nil
+}
+
+func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
+	type localItemName struct {
+		Name string `yaml:"name"`
+	}
+
+	_, fileName := filepath.Split(path)
+
+	item := &Item{
+		hub:      h,
+		Name:     info.fname,
+		Stage:    info.stage,
+		Type:     info.ftype,
+		FileName: fileName,
+		State: ItemState{
+			LocalPath: path,
+			Installed: true,
+			UpToDate:  true,
+		},
+	}
+
+	// try to read the name from the file
+	itemName := localItemName{}
+
+	itemContent, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s: %w", path, err)
+	}
+
+	err = yaml.Unmarshal(itemContent, &itemName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
+	}
+
+	if itemName.Name != "" {
+		item.Name = itemName.Name
+	}
+
+	return item, nil
+}
+
+func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
+	hubpath := ""
+
+	if err != nil {
+		log.Debugf("while syncing hub dir: %s", err)
+		// there is a path error, we ignore the file
+		return nil
+	}
+
+	// only happens if the current working directory was removed (!)
+	path, err = filepath.Abs(path)
+	if err != nil {
+		return err
+	}
+
+	// we only care about YAML files
+	if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) {
+		return nil
+	}
+
+	info, err := h.getItemFileInfo(path)
+	if err != nil {
+		return err
+	}
+
+	// non symlinks are local user files or hub files
+	if f.Type()&os.ModeSymlink == 0 {
+		log.Tracef("%s is not a symlink", path)
+
+		if !info.inhub {
+			log.Tracef("%s is a local file, skip", path)
+			item, err := newLocalItem(h, path, info)
+			if err != nil {
+				return err
+			}
+			h.Items[info.ftype][item.Name] = item
+
+			return nil
+		}
+	} else {
+		hubpath, err = linkTarget(path)
+		if err != nil {
+			return err
+		}
+
+		if hubpath == "" {
+			// target does not exist, the user might have removed the file
+			// or switched to a hub branch without it
+			return nil
+		}
+	}
+
+	// try to find which configuration item it is
+	log.Tracef("check [%s] of %s", info.fname, info.ftype)
+
+	for name, item := range h.Items[info.ftype] {
+		if info.fname != item.FileName {
+			continue
+		}
+
+		if item.Stage != info.stage {
+			continue
+		}
+
+		// if we are walking hub dir, just mark present files as downloaded
+		if info.inhub {
+			// wrong author
+			if info.fauthor != item.Author {
+				continue
+			}
+
+			// not the item we're looking for
+			if !item.validPath(info.fauthor, info.fname) {
+				continue
+			}
+
+			src, err := item.downloadPath()
+			if err != nil {
+				return err
+			}
+
+			if path == src {
+				log.Tracef("marking %s as downloaded", item.Name)
+				item.State.Downloaded = true
+			}
+		} else if !hasPathSuffix(hubpath, item.RemotePath) {
+			// wrong file
+			// <type>/<stage>/<author>/<name>.yaml
+			continue
+		}
+
+		err := item.setVersionState(path, info.inhub)
+		if err != nil {
+			return err
+		}
+
+		h.Items[info.ftype][name] = item
+
+		return nil
+	}
+
+	log.Infof("Ignoring file %s of type %s", path, info.ftype)
+
+	return nil
+}
+
+// checkSubItemVersions checks for the presence, taint and version state of sub-items.
+func (i *Item) checkSubItemVersions() error {
+	if !i.HasSubItems() {
+		return nil
+	}
+
+	if i.versionStatus() != versionUpToDate {
+		log.Debugf("%s dependencies not checked: not up-to-date", i.Name)
+		return nil
+	}
+
+	// ensure all the sub-items are installed, or tag the parent as tainted
+	log.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed)
+
+	for _, sub := range i.SubItems() {
+		log.Tracef("check %s installed:%t", sub.Name, sub.State.Installed)
+
+		if !i.State.Installed {
+			continue
+		}
+
+		if err := sub.checkSubItemVersions(); err != nil {
+			if sub.State.Tainted {
+				i.State.Tainted = true
+			}
+
+			return fmt.Errorf("dependency of %s: sub collection %s is broken: %w", i.Name, sub.Name, err)
+		}
+
+		if sub.State.Tainted {
+			i.State.Tainted = true
+			return fmt.Errorf("%s is tainted because %s:%s is tainted", i.Name, sub.Type, sub.Name)
+		}
+
+		if !sub.State.Installed && i.State.Installed {
+			i.State.Tainted = true
+			return fmt.Errorf("%s is tainted because %s:%s is missing", i.Name, sub.Type, sub.Name)
+		}
+
+		if !sub.State.UpToDate {
+			i.State.UpToDate = false
+			return fmt.Errorf("dependency of %s: outdated %s:%s", i.Name, sub.Type, sub.Name)
+		}
+
+		log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate)
+	}
+
+	return nil
+}
+
+// syncDir scans a directory for items, and updates the Hub state accordingly.
+func (h *Hub) syncDir(dir string) error {
+	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
+	for _, scan := range ItemTypes {
+		// cpath: top-level item directory, either downloaded or installed items.
+		// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...
+		cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
+		if err != nil {
+			log.Errorf("failed %s: %s", cpath, err)
+			continue
+		}
+
+		// explicit check for non existing directory, avoid spamming log.Debug
+		if _, err = os.Stat(cpath); os.IsNotExist(err) {
+			log.Tracef("directory %s doesn't exist, skipping", cpath)
+			continue
+		}
+
+		if err = filepath.WalkDir(cpath, h.itemVisit); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// insert a string in a sorted slice, case insensitive, and return the new slice.
+func insertInOrderNoCase(sl []string, value string) []string {
+	i := sort.Search(len(sl), func(i int) bool {
+		return strings.ToLower(sl[i]) >= strings.ToLower(value)
+	})
+
+	return append(sl[:i], append([]string{value}, sl[i:]...)...)
+}
+
+// localSync updates the hub state with downloaded, installed and local items.
+func (h *Hub) localSync() error {
+	err := h.syncDir(h.local.InstallDir)
+	if err != nil {
+		return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
+	}
+
+	if err = h.syncDir(h.local.HubDir); err != nil {
+		return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
+	}
+
+	warnings := make([]string, 0)
+
+	for _, item := range h.Items[COLLECTIONS] {
+		// check for cyclic dependencies
+		subs, err := item.descendants()
+		if err != nil {
+			return err
+		}
+
+		// populate the sub- and sub-sub-items with the collections they belong to
+		for _, sub := range subs {
+			sub.State.BelongsToCollections = insertInOrderNoCase(sub.State.BelongsToCollections, item.Name)
+		}
+
+		if !item.State.Installed {
+			continue
+		}
+
+		vs := item.versionStatus()
+		switch vs {
+		case versionUpToDate: // latest
+			if err := item.checkSubItemVersions(); err != nil {
+				warnings = append(warnings, err.Error())
+			}
+		case versionUpdateAvailable: // not up-to-date
+			warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
+		case versionFuture:
+			warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
+		case versionUnknown:
+			if !item.IsLocal() {
+				warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
+			}
+		}
+
+		log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
+	}
+
+	h.Warnings = warnings
+
+	return nil
+}
+
+func (i *Item) setVersionState(path string, inhub bool) error {
+	var err error
+
+	i.State.LocalHash, err = getSHA256(path)
+	if err != nil {
+		return fmt.Errorf("failed to get sha256 of %s: %w", path, err)
+	}
+
+	// let's reverse sort the versions to deal with hash collisions (#154)
+	versions := make([]string, 0, len(i.Versions))
+	for k := range i.Versions {
+		versions = append(versions, k)
+	}
+
+	versions, err = sortedVersions(versions)
+	if err != nil {
+		return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err)
+	}
+
+	i.State.LocalVersion = "?"
+
+	for _, version := range versions {
+		if i.Versions[version].Digest == i.State.LocalHash {
+			i.State.LocalVersion = version
+			break
+		}
+	}
+
+	if i.State.LocalVersion == "?" {
+		log.Tracef("got tainted match for %s: %s", i.Name, path)
+
+		if !inhub {
+			i.State.LocalPath = path
+			i.State.Installed = true
+		}
+
+		i.State.UpToDate = false
+		i.State.Tainted = true
+
+		return nil
+	}
+
+	// we got an exact match, update struct
+
+	i.State.Downloaded = true
+
+	if !inhub {
+		log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version)
+		i.State.LocalPath = path
+		i.State.Tainted = false
+		// if we're walking the hub, present file doesn't means installed file
+		i.State.Installed = true
+	}
+
+	if i.State.LocalVersion == i.Version {
+		log.Tracef("%s is up-to-date", i.Name)
+		i.State.UpToDate = true
+	}
+
+	return nil
+}

pkg/hubtest/coverage.go

@@ -5,173 +5,194 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
-	"sort"
 	"strings"
 
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-type ParserCoverage struct {
-	Parser     string
+type Coverage struct {
+	Name       string
 	TestsCount int
 	PresentIn  map[string]bool //poorman's set
 }
 
-type ScenarioCoverage struct {
-	Scenario   string
-	TestsCount int
-	PresentIn  map[string]bool
-}
-
-func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
-	var coverage []ParserCoverage
-	if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok {
-		return coverage, fmt.Errorf("no parsers in hub index")
+func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
+	if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
+		return nil, fmt.Errorf("no parsers in hub index")
 	}
-	//populate from hub, iterate in alphabetical order
-	var pkeys []string
-	for pname := range h.HubIndex.Data[cwhub.PARSERS] {
-		pkeys = append(pkeys, pname)
-	}
-	sort.Strings(pkeys)
-	for _, pname := range pkeys {
-		coverage = append(coverage, ParserCoverage{
-			Parser:     pname,
+
+	// populate from hub, iterate in alphabetical order
+	pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS])
+	coverage := make([]Coverage, len(pkeys))
+
+	for i, name := range pkeys {
+		coverage[i] = Coverage{
+			Name:       name,
 			TestsCount: 0,
 			PresentIn:  make(map[string]bool),
-		})
+		}
 	}
 
-	//parser the expressions a-la-oneagain
+	// parser the expressions a-la-oneagain
 	passerts, err := filepath.Glob(".tests/*/parser.assert")
 	if err != nil {
-		return coverage, fmt.Errorf("while find parser asserts : %s", err)
+		return nil, fmt.Errorf("while find parser asserts : %s", err)
 	}
+
 	for _, assert := range passerts {
 		file, err := os.Open(assert)
 		if err != nil {
-			return coverage, fmt.Errorf("while reading %s : %s", assert, err)
+			return nil, fmt.Errorf("while reading %s : %s", assert, err)
 		}
+
 		scanner := bufio.NewScanner(file)
 		for scanner.Scan() {
-			assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
 			line := scanner.Text()
 			log.Debugf("assert line : %s", line)
-			match := assertLine.FindStringSubmatch(line)
+
+			match := parserResultRE.FindStringSubmatch(line)
 			if len(match) == 0 {
 				log.Debugf("%s doesn't match", line)
 				continue
 			}
-			sidx := assertLine.SubexpIndex("parser")
+
+			sidx := parserResultRE.SubexpIndex("parser")
 			capturedParser := match[sidx]
+
 			for idx, pcover := range coverage {
-				if pcover.Parser == capturedParser {
+				if pcover.Name == capturedParser {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
-				parserNameSplit := strings.Split(pcover.Parser, "/")
+
+				parserNameSplit := strings.Split(pcover.Name, "/")
 				parserNameOnly := parserNameSplit[len(parserNameSplit)-1]
+
 				if parserNameOnly == capturedParser {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
+
 				capturedParserSplit := strings.Split(capturedParser, "/")
 				capturedParserName := capturedParserSplit[len(capturedParserSplit)-1]
+
 				if capturedParserName == parserNameOnly {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
+
 				if capturedParserName == parserNameOnly+"-logs" {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
 			}
 		}
+
 		file.Close()
 	}
+
 	return coverage, nil
 }
 
-func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) {
-	var coverage []ScenarioCoverage
-	if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok {
-		return coverage, fmt.Errorf("no scenarios in hub index")
-	}
-	//populate from hub, iterate in alphabetical order
-	var pkeys []string
-	for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] {
-		pkeys = append(pkeys, scenarioName)
+func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
+	if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
+		return nil, fmt.Errorf("no scenarios in hub index")
 	}
-	sort.Strings(pkeys)
-	for _, scenarioName := range pkeys {
-		coverage = append(coverage, ScenarioCoverage{
-			Scenario:   scenarioName,
+
+	// populate from hub, iterate in alphabetical order
+	pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS])
+	coverage := make([]Coverage, len(pkeys))
+
+	for i, name := range pkeys {
+		coverage[i] = Coverage{
+			Name:       name,
 			TestsCount: 0,
 			PresentIn:  make(map[string]bool),
-		})
+		}
 	}
 
-	//parser the expressions a-la-oneagain
+	// parser the expressions a-la-oneagain
 	passerts, err := filepath.Glob(".tests/*/scenario.assert")
 	if err != nil {
-		return coverage, fmt.Errorf("while find scenario asserts : %s", err)
+		return nil, fmt.Errorf("while find scenario asserts : %s", err)
 	}
+
+
 	for _, assert := range passerts {
 		file, err := os.Open(assert)
 		if err != nil {
-			return coverage, fmt.Errorf("while reading %s : %s", assert, err)
+			return nil, fmt.Errorf("while reading %s : %s", assert, err)
 		}
+
 		scanner := bufio.NewScanner(file)
 		for scanner.Scan() {
-			assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
 			line := scanner.Text()
 			log.Debugf("assert line : %s", line)
-			match := assertLine.FindStringSubmatch(line)
+			match := scenarioResultRE.FindStringSubmatch(line)
+
 			if len(match) == 0 {
 				log.Debugf("%s doesn't match", line)
 				continue
 			}
-			sidx := assertLine.SubexpIndex("scenario")
-			scanner_name := match[sidx]
+
+			sidx := scenarioResultRE.SubexpIndex("scenario")
+			scannerName := match[sidx]
+
 			for idx, pcover := range coverage {
-				if pcover.Scenario == scanner_name {
+				if pcover.Name == scannerName {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
-				scenarioNameSplit := strings.Split(pcover.Scenario, "/")
+
+				scenarioNameSplit := strings.Split(pcover.Name, "/")
 				scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1]
-				if scenarioNameOnly == scanner_name {
+
+				if scenarioNameOnly == scannerName {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
-				fixedProbingWord := strings.ReplaceAll(pcover.Scenario, "probbing", "probing")
-				fixedProbingAssert := strings.ReplaceAll(scanner_name, "probbing", "probing")
+
+				fixedProbingWord := strings.ReplaceAll(pcover.Name, "probbing", "probing")
+				fixedProbingAssert := strings.ReplaceAll(scannerName, "probbing", "probing")
+
 				if fixedProbingWord == fixedProbingAssert {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
-				if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name {
+
+				if fmt.Sprintf("%s-detection", pcover.Name) == scannerName {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
+
 				if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert {
 					coverage[idx].TestsCount++
 					coverage[idx].PresentIn[assert] = true
+
 					continue
 				}
 			}
 		}
 		file.Close()
 	}
+
 	return coverage, nil
 }

pkg/hubtest/hubtest.go

@@ -6,6 +6,7 @@ import (
 	"os/exec"
 	"path/filepath"
 
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -18,7 +19,7 @@ type HubTest struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *HubIndex
+	HubIndex               *cwhub.Hub
 	Tests                  []*HubTestItem
 }
 
@@ -29,42 +30,44 @@ const (
 )
 
 func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) {
-	var err error
-
-	hubPath, err = filepath.Abs(hubPath)
+	hubPath, err := filepath.Abs(hubPath)
 	if err != nil {
 		return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err)
 	}
+
 	// we can't use hubtest without the hub
-	if _, err := os.Stat(hubPath); os.IsNotExist(err) {
+	if _, err = os.Stat(hubPath); os.IsNotExist(err) {
 		return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath)
 	}
+
 	HubTestPath := filepath.Join(hubPath, "./.tests/")
 
 	// we can't use hubtest without crowdsec binary
-	if _, err := exec.LookPath(crowdsecPath); err != nil {
-		if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) {
+	if _, err = exec.LookPath(crowdsecPath); err != nil {
+		if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) {
 			return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath)
 		}
 	}
 
 	// we can't use hubtest without cscli binary
-	if _, err := exec.LookPath(cscliPath); err != nil {
-		if _, err := os.Stat(cscliPath); os.IsNotExist(err) {
+	if _, err = exec.LookPath(cscliPath); err != nil {
+		if _, err = os.Stat(cscliPath); os.IsNotExist(err) {
 			return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath)
 		}
 	}
 
 	hubIndexFile := filepath.Join(hubPath, ".index.json")
-	bidx, err := os.ReadFile(hubIndexFile)
-	if err != nil {
-		return HubTest{}, fmt.Errorf("unable to read index file: %s", err)
+
+	local := &csconfig.LocalHubCfg{
+		HubDir:         hubPath,
+		HubIndexFile:   hubIndexFile,
+		InstallDir:     HubTestPath,
+		InstallDataDir: HubTestPath,
 	}
 
-	// load hub index
-	hubIndex, err := cwhub.LoadPkgIndex(bidx)
+	hub, err := cwhub.NewHub(local, nil, false)
 	if err != nil {
-		return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err)
+		return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
 	}
 
 	templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile)
@@ -80,16 +83,18 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 		TemplateConfigPath:     templateConfigFilePath,
 		TemplateProfilePath:    templateProfilePath,
 		TemplateSimulationPath: templateSimulationPath,
-		HubIndex:               &HubIndex{Data: hubIndex},
+		HubIndex:               hub,
 	}, nil
 }
 
 func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) {
 	HubTestItem := &HubTestItem{}
+
 	testItem, err := NewTest(name, h)
 	if err != nil {
 		return HubTestItem, err
 	}
+
 	h.Tests = append(h.Tests, testItem)
 
 	return testItem, nil
@@ -108,5 +113,6 @@ func (h *HubTest) LoadAllTests() error {
 			}
 		}
 	}
+
 	return nil
 }

pkg/hubtest/hubtest_item.go

@@ -7,11 +7,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
-	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
 )
 
 type HubTestItemConfig struct {
@@ -25,10 +26,6 @@ type HubTestItemConfig struct {
 	OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
 }
 
-type HubIndex struct {
-	Data map[string]map[string]cwhub.Item
-}
-
 type HubTestItem struct {
 	Name string
 	Path string
@@ -43,7 +40,7 @@ type HubTestItem struct {
 	RuntimeConfigFilePath     string
 	RuntimeProfileFilePath    string
 	RuntimeSimulationFilePath string
-	RuntimeHubConfig          *csconfig.Hub
+	RuntimeHubConfig          *csconfig.LocalHubCfg
 
 	ResultsPath          string
 	ParserResultFile     string
@@ -56,7 +53,7 @@ type HubTestItem struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *HubIndex
+	HubIndex               *cwhub.Hub
 
 	Config *HubTestItemConfig
 
@@ -80,8 +77,6 @@ const (
 	BucketPourResultFileName = "bucketpour-dump.yaml"
 )
 
-var crowdsecPatternsFolder = csconfig.DefaultConfigPath("patterns")
-
 func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 	testPath := filepath.Join(hubTest.HubTestPath, name)
 	runtimeFolder := filepath.Join(testPath, "runtime")
@@ -91,10 +86,12 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 
 	// read test configuration file
 	configFileData := &HubTestItemConfig{}
+
 	yamlFile, err := os.ReadFile(configFilePath)
 	if err != nil {
 		log.Printf("no config file found in '%s': %v", testPath, err)
 	}
+
 	err = yaml.Unmarshal(yamlFile, configFileData)
 	if err != nil {
 		return nil, fmt.Errorf("unmarshal: %v", err)
@@ -105,6 +102,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 
 	scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName)
 	ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath)
+
 	return &HubTestItem{
 		Name:                      name,
 		Path:                      testPath,
@@ -121,7 +119,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 		ParserResultFile:          filepath.Join(resultPath, ParserResultFileName),
 		ScenarioResultFile:        filepath.Join(resultPath, ScenarioResultFileName),
 		BucketPourResultFile:      filepath.Join(resultPath, BucketPourResultFileName),
-		RuntimeHubConfig: &csconfig.Hub{
+		RuntimeHubConfig: &csconfig.LocalHubCfg{
 			HubDir:         runtimeHubFolder,
 			HubIndexFile:   hubTest.HubIndexFile,
 			InstallDir:     runtimeFolder,
@@ -147,23 +145,25 @@ func (t *HubTestItem) InstallHub() error {
 		if parser == "" {
 			continue
 		}
-		var parserDirDest string
-		if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok {
+
+		if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
 			parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
 			}
+
 			parserFileName := filepath.Base(parserSource)
 
 			// runtime/hub/parsers/s00-raw/crowdsecurity/
 			hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath))
 
 			// runtime/parsers/s00-raw/
-			parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
+			parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
 
 			if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil {
 				return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err)
 			}
+
 			if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
 				return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
 			}
@@ -204,7 +204,7 @@ func (t *HubTestItem) InstallHub() error {
 					//return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
 				}
 
-				parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
+				parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
 				if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
 					continue
 					//return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
@@ -231,23 +231,25 @@ func (t *HubTestItem) InstallHub() error {
 		if scenario == "" {
 			continue
 		}
-		var scenarioDirDest string
-		if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok {
+
+		if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
 			scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
 			}
+
 			scenarioFileName := filepath.Base(scenarioSource)
 
 			// runtime/hub/scenarios/crowdsecurity/
 			hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath))
 
 			// runtime/parsers/scenarios/
-			scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
+			scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
 
 			if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil {
 				return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err)
 			}
+
 			if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
 				return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
 			}
@@ -275,7 +277,7 @@ func (t *HubTestItem) InstallHub() error {
 					//return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario)
 				}
 
-				scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
+				scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
 				if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
 					return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
 				}
@@ -300,23 +302,25 @@ func (t *HubTestItem) InstallHub() error {
 		if postoverflow == "" {
 			continue
 		}
-		var postoverflowDirDest string
-		if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok {
+
+		if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
 			postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
 			}
+
 			postoverflowFileName := filepath.Base(postoverflowSource)
 
 			// runtime/hub/postoverflows/s00-enrich/crowdsecurity/
 			hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath))
 
 			// runtime/postoverflows/s00-enrich
-			postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
+			postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
 
 			if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil {
 				return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err)
 			}
+
 			if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
 				return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
 			}
@@ -357,7 +361,7 @@ func (t *HubTestItem) InstallHub() error {
 					//return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
 				}
 
-				postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
+				postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
 				if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
 					continue
 					//return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
@@ -384,10 +388,12 @@ func (t *HubTestItem) InstallHub() error {
 			Filter:  "1==1",
 			Statics: t.Config.OverrideStatics,
 		}
+
 		b, err := yaml.Marshal(n)
 		if err != nil {
 			return fmt.Errorf("unable to marshal overrides: %s", err)
 		}
+
 		tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath)
 		if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil {
 			return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err)
@@ -395,40 +401,43 @@ func (t *HubTestItem) InstallHub() error {
 	}
 
 	// load installed hub
-	err := cwhub.GetHubIdx(t.RuntimeHubConfig)
+	hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false)
 	if err != nil {
-		log.Fatalf("can't local sync the hub: %+v", err)
+		log.Fatal(err)
 	}
 
 	// install data for parsers if needed
-	ret := cwhub.GetItemMap(cwhub.PARSERS)
+	ret := hub.GetItemMap(cwhub.PARSERS)
 	for parserName, item := range ret {
-		if item.Installed {
-			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+		if item.State.Installed {
+			if err := item.DownloadDataIfNeeded(true); err != nil {
 				return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
 			}
+
 			log.Debugf("parser '%s' installed successfully in runtime environment", parserName)
 		}
 	}
 
 	// install data for scenarios if needed
-	ret = cwhub.GetItemMap(cwhub.SCENARIOS)
+	ret = hub.GetItemMap(cwhub.SCENARIOS)
 	for scenarioName, item := range ret {
-		if item.Installed {
-			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+		if item.State.Installed {
+			if err := item.DownloadDataIfNeeded(true); err != nil {
 				return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
 			}
+
 			log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName)
 		}
 	}
 
 	// install data for postoverflows if needed
-	ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW)
+	ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
 	for postoverflowName, item := range ret {
-		if item.Installed {
-			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+		if item.State.Installed {
+			if err := item.DownloadDataIfNeeded(true); err != nil {
 				return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
 			}
+
 			log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName)
 		}
 	}
@@ -455,51 +464,53 @@ func (t *HubTestItem) Run() error {
 	}
 
 	// create runtime folder
-	if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
+	if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
 		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
 	}
 
 	// create runtime data folder
-	if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
+	if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
 		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
 	}
 
 	// create runtime hub folder
-	if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
+	if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
 		return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
 	}
 
-	if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
+	if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
 		return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
 	}
 
 	// create results folder
-	if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
+	if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
 		return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
 	}
 
 	// copy template config file to runtime folder
-	if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
+	if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
 		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
 	}
 
 	// copy template profile file to runtime folder
-	if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
+	if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
 		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
 	}
 
 	// copy template simulation file to runtime folder
-	if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
+	if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
 		return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
 	}
 
+	crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns")
+
 	// copy template patterns folder to runtime folder
-	if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
+	if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
 		return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
 	}
 
 	// install the hub in the runtime folder
-	if err := t.InstallHub(); err != nil {
+	if err = t.InstallHub(); err != nil {
 		return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
 	}
 
@@ -507,7 +518,7 @@ func (t *HubTestItem) Run() error {
 	logType := t.Config.LogType
 	dsn := fmt.Sprintf("file://%s", logFile)
 
-	if err := os.Chdir(testPath); err != nil {
+	if err = os.Chdir(testPath); err != nil {
 		return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
 	}
 
@@ -515,6 +526,7 @@ func (t *HubTestItem) Run() error {
 	if err != nil {
 		return fmt.Errorf("unable to stat log file '%s': %s", logFile, err)
 	}
+
 	if logFileStat.Size() == 0 {
 		return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile)
 	}
@@ -522,6 +534,7 @@ func (t *HubTestItem) Run() error {
 	cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
 	cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
 	log.Debugf("%s", cscliRegisterCmd.String())
+
 	output, err := cscliRegisterCmd.CombinedOutput()
 	if err != nil {
 		if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
@@ -531,16 +544,20 @@ func (t *HubTestItem) Run() error {
 	}
 
 	cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"}
+
 	for labelKey, labelValue := range t.Config.Labels {
 		arg := fmt.Sprintf("%s:%s", labelKey, labelValue)
 		cmdArgs = append(cmdArgs, "-label", arg)
 	}
+
 	crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...)
 	log.Debugf("%s", crowdsecCmd.String())
 	output, err = crowdsecCmd.CombinedOutput()
+
 	if log.GetLevel() >= log.DebugLevel || err != nil {
 		fmt.Println(string(output))
 	}
+
 	if err != nil {
 		return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err)
 	}
@@ -557,8 +574,10 @@ func (t *HubTestItem) Run() error {
 			if err != nil {
 				return err
 			}
+
 			parserAssertFile.Close()
 		}
+
 		assertFileStat, err := os.Stat(t.ParserAssert.File)
 		if err != nil {
 			return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err)
@@ -569,6 +588,7 @@ func (t *HubTestItem) Run() error {
 			if err != nil {
 				return fmt.Errorf("couldn't generate assertion: %s", err)
 			}
+
 			t.ParserAssert.AutoGenAssertData = assertData
 			t.ParserAssert.AutoGenAssert = true
 		} else {
@@ -580,12 +600,15 @@ func (t *HubTestItem) Run() error {
 
 	// assert scenarios
 	nbScenario := 0
+
 	for _, scenario := range t.Config.Scenarios {
 		if scenario == "" {
 			continue
 		}
-		nbScenario += 1
+
+		nbScenario++
 	}
+
 	if nbScenario > 0 {
 		_, err := os.Stat(t.ScenarioAssert.File)
 		if os.IsNotExist(err) {
@@ -593,8 +616,10 @@ func (t *HubTestItem) Run() error {
 			if err != nil {
 				return err
 			}
+
 			scenarioAssertFile.Close()
 		}
+
 		assertFileStat, err := os.Stat(t.ScenarioAssert.File)
 		if err != nil {
 			return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err)
@@ -605,6 +630,7 @@ func (t *HubTestItem) Run() error {
 			if err != nil {
 				return fmt.Errorf("couldn't generate assertion: %s", err)
 			}
+
 			t.ScenarioAssert.AutoGenAssertData = assertData
 			t.ScenarioAssert.AutoGenAssert = true
 		} else {

pkg/hubtest/parser_assert.go

@@ -5,13 +5,11 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"regexp"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/antonmedv/expr"
-	"github.com/antonmedv/expr/vm"
 	"github.com/enescakir/emoji"
 	"github.com/fatih/color"
 	diff "github.com/r3labs/diff/v2"
@@ -43,10 +41,10 @@ type ParserResult struct {
 	Evt     types.Event
 	Success bool
 }
+
 type ParserResults map[string]map[string][]ParserResult
 
 func NewParserAssert(file string) *ParserAssert {
-
 	ParserAssert := &ParserAssert{
 		File:          file,
 		NbAssert:      0,
@@ -55,6 +53,7 @@ func NewParserAssert(file string) *ParserAssert {
 		AutoGenAssert: false,
 		TestData:      &ParserResults{},
 	}
+
 	return ParserAssert
 }
 
@@ -63,22 +62,24 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
 	ret := p.AutoGenParserAssert()
+
 	return ret, nil
 }
 
 func (p *ParserAssert) LoadTest(filename string) error {
-	var err error
 	parserDump, err := LoadParserDump(filename)
 	if err != nil {
 		return fmt.Errorf("loading parser dump file: %+v", err)
 	}
+
 	p.TestData = parserDump
+
 	return nil
 }
 
 func (p *ParserAssert) AssertFile(testFile string) error {
-
 	file, err := os.Open(p.File)
 
 	if err != nil {
@@ -88,19 +89,26 @@ func (p *ParserAssert) AssertFile(testFile string) error {
 	if err := p.LoadTest(testFile); err != nil {
 		return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
 	}
+
 	scanner := bufio.NewScanner(file)
 	scanner.Split(bufio.ScanLines)
+
 	nbLine := 0
+
 	for scanner.Scan() {
-		nbLine += 1
+		nbLine++
+
 		if scanner.Text() == "" {
 			continue
 		}
+
 		ok, err := p.Run(scanner.Text())
 		if err != nil {
 			return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
 		}
-		p.NbAssert += 1
+
+		p.NbAssert++
+
 		if !ok {
 			log.Debugf("%s is FALSE", scanner.Text())
 			failedAssert := &AssertFail{
@@ -109,37 +117,43 @@ func (p *ParserAssert) AssertFile(testFile string) error {
 				Expression: scanner.Text(),
 				Debug:      make(map[string]string),
 			}
-			variableRE := regexp.MustCompile(`(?P<variable>[^  =]+) == .*`)
+
 			match := variableRE.FindStringSubmatch(scanner.Text())
 			variable := ""
+
 			if len(match) == 0 {
 				log.Infof("Couldn't get variable of line '%s'", scanner.Text())
 				variable = scanner.Text()
 			} else {
 				variable = match[1]
 			}
+
 			result, err := p.EvalExpression(variable)
 			if err != nil {
 				log.Errorf("unable to evaluate variable '%s': %s", variable, err)
 				continue
 			}
+
 			failedAssert.Debug[variable] = result
 			p.Fails = append(p.Fails, *failedAssert)
 
 			continue
 		}
 		//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
-
 	}
+
 	file.Close()
+
 	if p.NbAssert == 0 {
 		assertData, err := p.AutoGenFromFile(testFile)
 		if err != nil {
 			return fmt.Errorf("couldn't generate assertion: %s", err)
 		}
+
 		p.AutoGenAssertData = assertData
 		p.AutoGenAssert = true
 	}
+
 	if len(p.Fails) == 0 {
 		p.Success = true
 	}
@@ -148,15 +162,14 @@ func (p *ParserAssert) AssertFile(testFile string) error {
 }
 
 func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
-	var err error
 	//debug doesn't make much sense with the ability to evaluate "on the fly"
 	//var debugFilter *exprhelpers.ExprDebugger
-	var runtimeFilter *vm.Program
 	var output interface{}
 
 	env := map[string]interface{}{"results": *p.TestData}
 
-	if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
+	runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
+	if err != nil {
 		log.Errorf("failed to compile '%s' : %s", expression, err)
 		return output, err
 	}
@@ -168,8 +181,10 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
 	if err != nil {
 		log.Warningf("running : %s", expression)
 		log.Warningf("runtime error : %s", err)
+
 		return output, fmt.Errorf("while running expression %s: %w", expression, err)
 	}
+
 	return output, nil
 }
 
@@ -178,10 +193,13 @@ func (p *ParserAssert) EvalExpression(expression string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
 	ret, err := yaml.Marshal(output)
+
 	if err != nil {
 		return "", err
 	}
+
 	return string(ret), nil
 }
 
@@ -190,6 +208,7 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
+
 	switch out := output.(type) {
 	case bool:
 		return out, nil
@@ -201,80 +220,89 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
 func Escape(val string) string {
 	val = strings.ReplaceAll(val, `\`, `\\`)
 	val = strings.ReplaceAll(val, `"`, `\"`)
+
 	return val
 }
 
 func (p *ParserAssert) AutoGenParserAssert() string {
 	//attempt to autogen parser asserts
-	var ret string
+	ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
+
+	//sort map keys for consistent order
+	stages := sortedMapKeys(*p.TestData)
 
-	//sort map keys for consistent ordre
-	var stages []string
-	for stage := range *p.TestData {
-		stages = append(stages, stage)
-	}
-	sort.Strings(stages)
-	ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
 	for _, stage := range stages {
 		parsers := (*p.TestData)[stage]
-		//sort map keys for consistent ordre
-		var pnames []string
-		for pname := range parsers {
-			pnames = append(pnames, pname)
-		}
-		sort.Strings(pnames)
+
+		//sort map keys for consistent order
+		pnames := sortedMapKeys(parsers)
+
 		for _, parser := range pnames {
 			presults := parsers[parser]
 			ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults))
+
 			for pidx, result := range presults {
 				ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success)
 
 				if !result.Success {
 					continue
 				}
+
 				for _, pkey := range sortedMapKeys(result.Evt.Parsed) {
 					pval := result.Evt.Parsed[pkey]
 					if pval == "" {
 						continue
 					}
+
 					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
 				}
+
 				for _, mkey := range sortedMapKeys(result.Evt.Meta) {
 					mval := result.Evt.Meta[mkey]
 					if mval == "" {
 						continue
 					}
+
 					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
 				}
+
 				for _, ekey := range sortedMapKeys(result.Evt.Enriched) {
 					eval := result.Evt.Enriched[ekey]
 					if eval == "" {
 						continue
 					}
+
 					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
 				}
+
 				for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) {
 					uval := result.Evt.Unmarshaled[ukey]
 					if uval == "" {
 						continue
 					}
+
 					base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey)
+
 					for _, line := range p.buildUnmarshaledAssert(base, uval) {
 						ret += line
 					}
 				}
+
 				ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted)
+
 				if result.Evt.WhitelistReason != "" {
 					ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason))
 				}
 			}
 		}
 	}
+
 	return ret
 }
 
 func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string {
 	ret := make([]string, 0)
+
 	switch val := eval.(type) {
 	case map[string]interface{}:
 		for k, v := range val {
@@ -297,12 +325,11 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s
 	default:
 		log.Warningf("unknown type '%T' for key '%s'", val, ekey)
 	}
+
 	return ret
 }
 
 func LoadParserDump(filepath string) (*ParserResults, error) {
-	var pdump ParserResults
-
 	dumpData, err := os.Open(filepath)
 	if err != nil {
 		return nil, err
@@ -314,18 +341,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
 		return nil, err
 	}
 
+	pdump := ParserResults{}
+
 	if err := yaml.Unmarshal(results, &pdump); err != nil {
 		return nil, err
 	}
 
 	/* we know that some variables should always be set,
 	let's check if they're present in last parser output of last stage */
-	stages := make([]string, 0, len(pdump))
-	for k := range pdump {
-		stages = append(stages, k)
-	}
-	sort.Strings(stages)
+
+	stages := sortedMapKeys(pdump)
+
 	var lastStage string
+
 	//Loop over stages to find last successful one with at least one parser
 	for i := len(stages) - 2; i >= 0; i-- {
 		if len(pdump[stages[i]]) != 0 {
@@ -333,11 +361,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
 			break
 		}
 	}
+
 	parsers := make([]string, 0, len(pdump[lastStage]))
+
 	for k := range pdump[lastStage] {
 		parsers = append(parsers, k)
 	}
+
 	sort.Strings(parsers)
+
+	if len(parsers) == 0 {
+		return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
+	}
+
 	lastParser := parsers[len(parsers)-1]
 
 	for idx, result := range pdump[lastStage][lastParser] {
@@ -357,47 +393,51 @@ type DumpOpts struct {
 	ShowNotOkParsers bool
 }
 
-func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts DumpOpts) {
+func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
 	//note : we can use line -> time as the unique identifier (of acquisition)
-
 	state := make(map[time.Time]map[string]map[string]ParserResult)
 	assoc := make(map[time.Time]string, 0)
 
-	for stage, parsers := range parser_results {
+	for stage, parsers := range parserResults {
 		for parser, results := range parsers {
-			for _, parser_res := range results {
-				evt := parser_res.Evt
+			for _, parserRes := range results {
+				evt := parserRes.Evt
 				if _, ok := state[evt.Line.Time]; !ok {
 					state[evt.Line.Time] = make(map[string]map[string]ParserResult)
 					assoc[evt.Line.Time] = evt.Line.Raw
 				}
+
 				if _, ok := state[evt.Line.Time][stage]; !ok {
 					state[evt.Line.Time][stage] = make(map[string]ParserResult)
 				}
-				state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success}
-			}
 
+				state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
+			}
 		}
 	}
 
-	for bname, evtlist := range bucket_pour {
+	for bname, evtlist := range bucketPour {
 		for _, evt := range evtlist {
 			if evt.Line.Raw == "" {
 				continue
 			}
+
 			//it might be bucket overflow being reprocessed, skip this
 			if _, ok := state[evt.Line.Time]; !ok {
 				state[evt.Line.Time] = make(map[string]map[string]ParserResult)
 				assoc[evt.Line.Time] = evt.Line.Raw
 			}
+
 			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
 			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
 			if _, ok := state[evt.Line.Time]["buckets"]; !ok {
 				state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
 			}
+
 			state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
 		}
 	}
+
 	yellow := color.New(color.FgYellow).SprintFunc()
 	red := color.New(color.FgRed).SprintFunc()
 	green := color.New(color.FgGreen).SprintFunc()
@@ -409,19 +449,25 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
 				continue
 			}
 		}
+
 		fmt.Printf("line: %s\n", rawstr)
+
 		skeys := make([]string, 0, len(state[tstamp]))
+
 		for k := range state[tstamp] {
 			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
 			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
 			if k == "buckets" {
 				continue
 			}
+
 			skeys = append(skeys, k)
 		}
+
 		sort.Strings(skeys)
-		//iterate stage
-		var prev_item types.Event
+
+		// iterate stage
+		var prevItem types.Event
 
 		for _, stage := range skeys {
 			parsers := state[tstamp][stage]
@@ -431,18 +477,16 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
 
 			fmt.Printf("\t%s %s\n", sep, stage)
 
-			pkeys := make([]string, 0, len(parsers))
-			for k := range parsers {
-				pkeys = append(pkeys, k)
-			}
-			sort.Strings(pkeys)
+			pkeys := sortedMapKeys(parsers)
 
 			for idx, parser := range pkeys {
 				res := parsers[parser].Success
 				sep := "├"
+
 				if idx == len(pkeys)-1 {
 					sep = "└"
 				}
+
 				created := 0
 				updated := 0
 				deleted := 0
@@ -451,16 +495,19 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
 				detailsDisplay := ""
 
 				if res {
-					changelog, _ := diff.Diff(prev_item, parsers[parser].Evt)
+					changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
 					for _, change := range changelog {
 						switch change.Type {
 						case "create":
 							created++
+
 							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
 						case "update":
 							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
+
 							if change.Path[0] == "Whitelisted" && change.To == true {
 								whitelisted = true
+
 								if whitelistReason == "" {
 									whitelistReason = parsers[parser].Evt.WhitelistReason
 								}
@@ -468,51 +515,64 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
 							updated++
 						case "delete":
 							deleted++
+
 							detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
 						}
 					}
-					prev_item = parsers[parser].Evt
+
+					prevItem = parsers[parser].Evt
 				}
 
 				if created > 0 {
 					changeStr += green(fmt.Sprintf("+%d", created))
 				}
+
 				if updated > 0 {
 					if len(changeStr) > 0 {
 						changeStr += " "
 					}
+
 					changeStr += yellow(fmt.Sprintf("~%d", updated))
 				}
+
 				if deleted > 0 {
 					if len(changeStr) > 0 {
 						changeStr += " "
 					}
+
 					changeStr += red(fmt.Sprintf("-%d", deleted))
 				}
+
 				if whitelisted {
 					if len(changeStr) > 0 {
 						changeStr += " "
 					}
+
 					changeStr += red("[whitelisted]")
 				}
+
 				if changeStr == "" {
 					changeStr = yellow("unchanged")
 				}
+
 				if res {
 					fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
+
 					if opts.Details {
 						fmt.Print(detailsDisplay)
 					}
 				} else if opts.ShowNotOkParsers {
 					fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
-
 				}
 			}
 		}
+
 		sep := "└"
+
 		if len(state[tstamp]["buckets"]) > 0 {
 			sep = "├"
 		}
+
 		//did the event enter the bucket pour phase ?
 		if _, ok := state[tstamp]["buckets"]["OK"]; ok {
 			fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
@@ -521,27 +581,35 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
 		} else {
 			fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
 		}
+
 		//now print bucket info
 		if len(state[tstamp]["buckets"]) > 0 {
 			fmt.Printf("\t├ Scenarios\n")
 		}
+
 		bnames := make([]string, 0, len(state[tstamp]["buckets"]))
+
 		for k := range state[tstamp]["buckets"] {
 			//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
 			//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
 			if k == "OK" {
 				continue
 			}
+
 			bnames = append(bnames, k)
 		}
+
 		sort.Strings(bnames)
+
 		for idx, bname := range bnames {
 			sep := "├"
 			if idx == len(bnames)-1 {
 				sep = "└"
 			}
+
 			fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
 		}
+
 		fmt.Println()
 	}
 }

pkg/hubtest/regexp.go

@@ -0,0 +1,11 @@
+package hubtest
+
+import (
+	"regexp"
+)
+
+var (
+	variableRE = regexp.MustCompile(`(?P<variable>[^  =]+) == .*`)
+	parserResultRE = regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
+	scenarioResultRE = regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
+)

pkg/hubtest/scenario_assert.go

@@ -5,12 +5,10 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"regexp"
 	"sort"
 	"strings"
 
 	"github.com/antonmedv/expr"
-	"github.com/antonmedv/expr/vm"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
@@ -42,6 +40,7 @@ func NewScenarioAssert(file string) *ScenarioAssert {
 		TestData:      &BucketResults{},
 		PourData:      &BucketPourInfo{},
 	}
+
 	return ScenarioAssert
 }
 
@@ -50,7 +49,9 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
 	ret := s.AutoGenScenarioAssert()
+
 	return ret, nil
 }
 
@@ -59,6 +60,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
 	if err != nil {
 		return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err)
 	}
+
 	s.TestData = bucketDump
 
 	if bucketpour != "" {
@@ -66,8 +68,10 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
 		if err != nil {
 			return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
 		}
+
 		s.PourData = pourDump
 	}
+
 	return nil
 }
 
@@ -81,19 +85,26 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
 	if err := s.LoadTest(testFile, ""); err != nil {
 		return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
 	}
+
 	scanner := bufio.NewScanner(file)
 	scanner.Split(bufio.ScanLines)
+
 	nbLine := 0
+
 	for scanner.Scan() {
-		nbLine += 1
+		nbLine++
+
 		if scanner.Text() == "" {
 			continue
 		}
+
 		ok, err := s.Run(scanner.Text())
 		if err != nil {
 			return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
 		}
-		s.NbAssert += 1
+
+		s.NbAssert++
+
 		if !ok {
 			log.Debugf("%s is FALSE", scanner.Text())
 			failedAssert := &AssertFail{
@@ -102,31 +113,38 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
 				Expression: scanner.Text(),
 				Debug:      make(map[string]string),
 			}
-			variableRE := regexp.MustCompile(`(?P<variable>[^ ]+) == .*`)
+
 			match := variableRE.FindStringSubmatch(scanner.Text())
+
 			if len(match) == 0 {
 				log.Infof("Couldn't get variable of line '%s'", scanner.Text())
 				continue
 			}
+
 			variable := match[1]
+
 			result, err := s.EvalExpression(variable)
 			if err != nil {
 				log.Errorf("unable to evaluate variable '%s': %s", variable, err)
 				continue
 			}
+
 			failedAssert.Debug[variable] = result
 			s.Fails = append(s.Fails, *failedAssert)
+
 			continue
 		}
 		//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
-
 	}
+
 	file.Close()
+
 	if s.NbAssert == 0 {
 		assertData, err := s.AutoGenFromFile(testFile)
 		if err != nil {
 			return fmt.Errorf("couldn't generate assertion: %s", err)
 		}
+
 		s.AutoGenAssertData = assertData
 		s.AutoGenAssert = true
 	}
@@ -139,15 +157,14 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
 }
 
 func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
-	var err error
 	//debug doesn't make much sense with the ability to evaluate "on the fly"
 	//var debugFilter *exprhelpers.ExprDebugger
-	var runtimeFilter *vm.Program
 	var output interface{}
 
 	env := map[string]interface{}{"results": *s.TestData}
 
-	if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
+	runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
+	if err != nil {
 		return nil, err
 	}
 	// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil {
@@ -161,8 +178,10 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
 	if err != nil {
 		log.Warningf("running : %s", expression)
 		log.Warningf("runtime error : %s", err)
+
 		return nil, fmt.Errorf("while running expression %s: %w", expression, err)
 	}
+
 	return output, nil
 }
 
@@ -171,10 +190,12 @@ func (s *ScenarioAssert) EvalExpression(expression string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+
 	ret, err := yaml.Marshal(output)
 	if err != nil {
 		return "", err
 	}
+
 	return string(ret), nil
 }
 
@@ -183,6 +204,7 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
 	if err != nil {
 		return false, err
 	}
+
 	switch out := output.(type) {
 	case bool:
 		return out, nil
@@ -192,9 +214,9 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
 }
 
 func (s *ScenarioAssert) AutoGenScenarioAssert() string {
-	//attempt to autogen parser asserts
-	var ret string
-	ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
+	// attempt to autogen scenario asserts
+	ret := fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
+
 	for eventIndex, event := range *s.TestData {
 		for ipSrc, source := range event.Overflow.Sources {
 			ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex)
@@ -203,15 +225,18 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string {
 			ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope)
 			ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value)
 		}
+
 		for evtIndex, evt := range event.Overflow.Alert.Events {
 			for _, meta := range evt.Meta {
 				ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value))
 			}
 		}
+
 		ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario)
 		ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation)
 		ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount)
 	}
+
 	return ret
 }
 
@@ -228,8 +253,6 @@ func (b BucketResults) Swap(i, j int) {
 }
 
 func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
-	var bucketDump BucketPourInfo
-
 	dumpData, err := os.Open(filepath)
 	if err != nil {
 		return nil, err
@@ -241,6 +264,8 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
 		return nil, err
 	}
 
+	var bucketDump BucketPourInfo
+
 	if err := yaml.Unmarshal(results, &bucketDump); err != nil {
 		return nil, err
 	}
@@ -249,8 +274,6 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
 }
 
 func LoadScenarioDump(filepath string) (*BucketResults, error) {
-	var bucketDump BucketResults
-
 	dumpData, err := os.Open(filepath)
 	if err != nil {
 		return nil, err
@@ -262,6 +285,8 @@ func LoadScenarioDump(filepath string) (*BucketResults, error) {
 		return nil, err
 	}
 
+	var bucketDump BucketResults
+
 	if err := yaml.Unmarshal(results, &bucketDump); err != nil {
 		return nil, err
 	}

pkg/hubtest/utils.go

@@ -12,7 +12,9 @@ func sortedMapKeys[V any](m map[string]V) []string {
 	for k := range m {
 		keys = append(keys, k)
 	}
+
 	sort.Strings(keys)
+
 	return keys
 }
 
@@ -22,7 +24,7 @@ func Copy(src string, dst string) error {
 		return err
 	}
 
-	err = os.WriteFile(dst, content, 0644)
+	err = os.WriteFile(dst, content, 0o644)
 	if err != nil {
 		return err
 	}
@@ -43,16 +45,20 @@ func checkPathNotContained(path string, subpath string) error {
 	}
 
 	current := absSubPath
+
 	for {
 		if current == absPath {
 			return fmt.Errorf("cannot copy a folder onto itself")
 		}
+
 		up := filepath.Dir(current)
 		if current == up {
 			break
 		}
+
 		current = up
 	}
+
 	return nil
 }
 

pkg/hubtest/utils_test.go

@@ -3,16 +3,16 @@ package hubtest
 import (
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestCheckPathNotContained(t *testing.T) {
-	assert.Nil(t, checkPathNotContained("/foo", "/bar"))
-	assert.Nil(t, checkPathNotContained("/foo/bar", "/foo"))
-	assert.Nil(t, checkPathNotContained("/foo/bar", "/"))
-	assert.Nil(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else"))
-	assert.Nil(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else"))
-	assert.NotNil(t, checkPathNotContained("/foo", "/foo/bar"))
-	assert.NotNil(t, checkPathNotContained("/", "/foo"))
-	assert.NotNil(t, checkPathNotContained("/", "/foo/bar/baz"))
+	require.NoError(t, checkPathNotContained("/foo", "/bar"))
+	require.NoError(t, checkPathNotContained("/foo/bar", "/foo"))
+	require.NoError(t, checkPathNotContained("/foo/bar", "/"))
+	require.NoError(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else"))
+	require.NoError(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else"))
+	require.Error(t, checkPathNotContained("/foo", "/foo/bar"))
+	require.Error(t, checkPathNotContained("/", "/foo"))
+	require.Error(t, checkPathNotContained("/", "/foo/bar/baz"))
 }

pkg/leakybucket/buckets_test.go

@@ -8,12 +8,14 @@ import (
 	"html/template"
 	"io"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -33,28 +35,45 @@ func TestBucket(t *testing.T) {
 		envSetting = os.Getenv("TEST_ONLY")
 		tomb       = &tomb.Tomb{}
 	)
-	err := exprhelpers.Init(nil)
+
+	testdata := "./tests"
+
+	hubCfg := &csconfig.LocalHubCfg{
+		HubDir: filepath.Join(testdata, "hub"),
+		HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
+		InstallDataDir: testdata,
+	}
+
+	hub, err := cwhub.NewHub(hubCfg, nil, false)
+	if err != nil {
+		t.Fatalf("failed to init hub: %s", err)
+	}
+
+	err = exprhelpers.Init(nil)
 	if err != nil {
 		log.Fatalf("exprhelpers init failed: %s", err)
 	}
 
 	if envSetting != "" {
-		if err := testOneBucket(t, envSetting, tomb); err != nil {
+		if err := testOneBucket(t, hub, envSetting, tomb); err != nil {
 			t.Fatalf("Test '%s' failed : %s", envSetting, err)
 		}
 	} else {
 		wg := new(sync.WaitGroup)
-		fds, err := os.ReadDir("./tests/")
+		fds, err := os.ReadDir(testdata)
 		if err != nil {
 			t.Fatalf("Unable to read test directory : %s", err)
 		}
 		for _, fd := range fds {
-			fname := "./tests/" + fd.Name()
+			if fd.Name() == "hub" {
+				continue
+			}
+			fname := filepath.Join(testdata, fd.Name())
 			log.Infof("Running test on %s", fname)
 			tomb.Go(func() error {
 				wg.Add(1)
 				defer wg.Done()
-				if err := testOneBucket(t, fname, tomb); err != nil {
+				if err := testOneBucket(t, hub, fname, tomb); err != nil {
 					t.Fatalf("Test '%s' failed : %s", fname, err)
 				}
 				return nil
@@ -76,7 +95,7 @@ func watchTomb(tomb *tomb.Tomb) {
 	}
 }
 
-func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
+func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) error {
 
 	var (
 		holders []BucketFactory
@@ -112,10 +131,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
 		files = append(files, x.Filename)
 	}
 
-	cscfg := &csconfig.CrowdsecServiceCfg{
-		DataDir: "tests",
-	}
-	holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false)
+	cscfg := &csconfig.CrowdsecServiceCfg{}
+	holders, response, err := LoadBuckets(cscfg, hub, files, tomb, buckets, false)
 	if err != nil {
 		t.Fatalf("failed loading bucket : %s", err)
 	}
@@ -123,7 +140,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
 		watchTomb(tomb)
 		return nil
 	})
-	if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) {
+	if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) {
 		return fmt.Errorf("tests from %s failed", dir)
 	}
 	return nil

pkg/leakybucket/manager_load.go

@@ -178,7 +178,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
 	return nil
 }
 
-func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
+func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
 	var (
 		ret      = []BucketFactory{}
 		response chan types.Event
@@ -211,7 +211,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
 				log.Tracef("End of yaml file")
 				break
 			}
-			bucketFactory.DataDir = cscfg.DataDir
+			bucketFactory.DataDir = hub.GetDataDir()
 			//check empty
 			if bucketFactory.Name == "" {
 				log.Errorf("Won't load nameless bucket")
@@ -234,7 +234,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
 			bucketFactory.Filename = filepath.Clean(f)
 			bucketFactory.BucketName = seed.Generate()
 			bucketFactory.ret = response
-			hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
+			hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
 			if err != nil {
 				log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
 			} else {
@@ -242,8 +242,8 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
 					bucketFactory.Simulated = cscfg.SimulationConfig.IsSimulated(hubItem.Name)
 				}
 				if hubItem != nil {
-					bucketFactory.ScenarioVersion = hubItem.LocalVersion
-					bucketFactory.hash = hubItem.LocalHash
+					bucketFactory.ScenarioVersion = hubItem.State.LocalVersion
+					bucketFactory.hash = hubItem.State.LocalHash
 				} else {
 					log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
 				}

pkg/leakybucket/tests/hub/index.json

@@ -0,0 +1 @@
+{}

pkg/parser/unix_parser.go

@@ -57,24 +57,25 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
 
 // Return new parsers
 // nodes and povfwnodes are already initialized in parser.LoadStages
-func NewParsers() *Parsers {
+func NewParsers(hub *cwhub.Hub) *Parsers {
 	parsers := &Parsers{
 		Ctx:             &UnixParserCtx{},
 		Povfwctx:        &UnixParserCtx{},
 		StageFiles:      make([]Stagefile, 0),
 		PovfwStageFiles: make([]Stagefile, 0),
 	}
-	for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
-		for _, hubParserItem := range cwhub.GetItemMap(itemType) {
-			if hubParserItem.Installed {
+
+	for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
+		for _, hubParserItem := range hub.GetItemMap(itemType) {
+			if hubParserItem.State.Installed {
 				stagefile := Stagefile{
-					Filename: hubParserItem.LocalPath,
+					Filename: hubParserItem.State.LocalPath,
 					Stage:    hubParserItem.Stage,
 				}
 				if itemType == cwhub.PARSERS {
 					parsers.StageFiles = append(parsers.StageFiles, stagefile)
 				}
-				if itemType == cwhub.PARSERS_OVFLW {
+				if itemType == cwhub.POSTOVERFLOWS {
 					parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
 				}
 			}
@@ -97,16 +98,16 @@ func NewParsers() *Parsers {
 func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
 	var err error
 
-	patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
+	patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/")
 	log.Infof("Loading grok library %s", patternsDir)
 	/* load base regexps for two grok parsers */
 	parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
-		"data": cConfig.Crowdsec.DataDir})
+		"data": cConfig.ConfigPaths.DataDir})
 	if err != nil {
 		return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
 	}
 	parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
-		"data": cConfig.Crowdsec.DataDir})
+		"data": cConfig.ConfigPaths.DataDir})
 	if err != nil {
 		return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
 	}
@@ -116,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
 	*/
 	log.Infof("Loading enrich plugins")
 
-	parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
+	parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
 	if err != nil {
 		return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
 	}

pkg/setup/install.go

@@ -10,7 +10,6 @@ import (
 	goccyyaml "github.com/goccy/go-yaml"
 	"gopkg.in/yaml.v3"
 
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -46,22 +45,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
 }
 
 // InstallHubItems installs the objects recommended in a setup file.
-func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error {
+func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
 	setupEnvelope, err := decodeSetup(input, false)
 	if err != nil {
 		return err
 	}
 
-	if err := csConfig.LoadHub(); err != nil {
-		return fmt.Errorf("loading hub: %w", err)
-	}
-
-	cwhub.SetHubBranch()
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("getting hub index: %w", err)
-	}
-
 	for _, setupItem := range setupEnvelope.Setup {
 		forceAction := false
 		downloadOnly := false
@@ -73,14 +62,19 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 
 		if len(install.Collections) > 0 {
 			for _, collection := range setupItem.Install.Collections {
+				item := hub.GetItem(cwhub.COLLECTIONS, collection)
+				if item == nil {
+					return fmt.Errorf("collection %s not found", collection)
+				}
+
 				if dryRun {
 					fmt.Println("dry-run: would install collection", collection)
 
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
-					return fmt.Errorf("while installing collection %s: %w", collection, err)
+				if err := item.Install(forceAction, downloadOnly); err != nil {
+					return fmt.Errorf("while installing collection %s: %w", item.Name, err)
 				}
 			}
 		}
@@ -93,8 +87,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
-					return fmt.Errorf("while installing parser %s: %w", parser, err)
+				item := hub.GetItem(cwhub.PARSERS, parser)
+				if item == nil {
+					return fmt.Errorf("parser %s not found", parser)
+				}
+
+				if err := item.Install(forceAction, downloadOnly); err != nil {
+					return fmt.Errorf("while installing parser %s: %w", item.Name, err)
 				}
 			}
 		}
@@ -107,8 +106,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
-					return fmt.Errorf("while installing scenario %s: %w", scenario, err)
+				item := hub.GetItem(cwhub.SCENARIOS, scenario)
+				if item == nil {
+					return fmt.Errorf("scenario %s not found", scenario)
+				}
+
+				if err := item.Install(forceAction, downloadOnly); err != nil {
+					return fmt.Errorf("while installing scenario %s: %w", item.Name, err)
 				}
 			}
 		}
@@ -121,8 +125,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
-					return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
+				item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow)
+				if item == nil {
+					return fmt.Errorf("postoverflow %s not found", postoverflow)
+				}
+
+				if err := item.Install(forceAction, downloadOnly); err != nil {
+					return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err)
 				}
 			}
 		}

test/bats/00_wait_for.bats

@@ -0,0 +1,71 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+@test "run a command and capture its stdout" {
+    run -0 wait-for seq 1 3
+    assert_output - <<-EOT
+	1
+	2
+	3
+	EOT
+}
+
+@test "run a command and capture its stderr" {
+    rune -0 wait-for sh -c 'seq 1 3 >&2'
+    assert_stderr - <<-EOT
+	1
+	2
+	3
+	EOT
+}
+
+@test "run a command until a pattern is found in stdout" {
+    run -0 wait-for --out "1[12]0" seq 1 200
+    assert_line --index 0 "1"
+    assert_line --index -1 "110"
+    refute_line "111"
+}
+
+@test "run a command until a pattern is found in stderr" {
+    rune -0 wait-for --err "10" sh -c 'seq 1 20 >&2'
+    assert_stderr - <<-EOT
+	1
+	2
+	3
+	4
+	5
+	6
+	7
+	8
+	9
+	10
+	EOT
+}
+
+@test "run a command with timeout (no match)" {
+    # when the process is terminated without a match, it returns
+    # 256 - 15 (SIGTERM) = 241
+    rune -241 wait-for --timeout 0.1 --out "10" sh -c 'echo 1; sleep 3; echo 2'
+    assert_line 1
+    # there may be more, but we don't care
+}
+
+@test "run a command with timeout (match)" {
+    # when the process is terminated with a match, return code is 128
+    rune -128 wait-for --timeout .4 --out "2" sh -c 'echo 1; sleep .1; echo 2; echo 3; echo 4; sleep 10'
+    assert_output - <<-EOT
+	1
+	2
+	EOT
+}
+

test/bats/01_crowdsec.bats

@@ -24,28 +24,22 @@ teardown() {
 #----------
 
 @test "crowdsec (usage)" {
-    rune -0 timeout 2s "${CROWDSEC}" -h
-    assert_stderr_line --regexp "Usage of .*:"
-
-    rune -0 timeout 2s "${CROWDSEC}" --help
-    assert_stderr_line --regexp "Usage of .*:"
+    rune -0 wait-for --out "Usage of " "${CROWDSEC}" -h
+    rune -0 wait-for --out "Usage of " "${CROWDSEC}" --help
 }
 
 @test "crowdsec (unknown flag)" {
-    rune -2 timeout 2s "${CROWDSEC}" --foobar
-    assert_stderr_line "flag provided but not defined: -foobar"
-    assert_stderr_line --regexp "Usage of .*"
+    rune -0 wait-for --err "flag provided but not defined: -foobar" "$CROWDSEC" --foobar
 }
 
 @test "crowdsec (unknown argument)" {
-    rune -2 timeout 2s "${CROWDSEC}" trololo
-    assert_stderr_line "argument provided but not defined: trololo"
-    assert_stderr_line --regexp "Usage of .*"
+    rune -0 wait-for --err "argument provided but not defined: trololo" "${CROWDSEC}" trololo
 }
 
 @test "crowdsec (no api and no agent)" {
-    rune -1 timeout 2s "${CROWDSEC}" -no-api -no-cs
-    assert_stderr_line --partial "You must run at least the API Server or crowdsec"
+    rune -0 wait-for \
+        --err "You must run at least the API Server or crowdsec" \
+        "${CROWDSEC}" -no-api -no-cs
 }
 
 @test "crowdsec - print error on exit" {
@@ -55,20 +49,22 @@ teardown() {
     assert_stderr --partial "unable to create database client: unknown database type 'meh'"
 }
 
-@test "crowdsec - bad configuration (empty/missing common section)" {
+@test "crowdsec - default logging configuration (empty/missing common section)" {
     config_set '.common={}'
-    rune -1 "${CROWDSEC}"
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "unable to load configuration: common section is empty"
 
     config_set 'del(.common)'
-    rune -1 "${CROWDSEC}"
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "unable to load configuration: common section is empty"
 }
 
 @test "CS_LAPI_SECRET not strong enough" {
-    CS_LAPI_SECRET=foo rune -1 timeout 2s "${CROWDSEC}"
+    CS_LAPI_SECRET=foo rune -1 wait-for "${CROWDSEC}"
     assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
 }
 
@@ -138,8 +134,8 @@ teardown() {
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
     rm -f "$ACQUIS_YAML"
 
-    rune -1 timeout 2s "${CROWDSEC}"
-    assert_stderr_line --partial "acquis.yaml: no such file or directory"
+    rune -1 wait-for "${CROWDSEC}"
+    assert_stderr --partial "acquis.yaml: no such file or directory"
 }
 
 @test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" {
@@ -151,7 +147,7 @@ teardown() {
     rm -f "$ACQUIS_DIR"
 
     config_set '.common.log_media="stdout"'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -1 wait-for "${CROWDSEC}"
     # check warning
     assert_stderr --partial "no acquisition file found"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
@@ -167,13 +163,15 @@ teardown() {
     config_set '.crowdsec_service.acquisition_dir=""'
 
     config_set '.common.log_media="stdout"'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -1 wait-for "${CROWDSEC}"
     # check warning
     assert_stderr --partial "no acquisition_path or acquisition_dir specified"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
+    config_set '.common.log_media="stdout"'
+
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
     config_set '.crowdsec_service.acquisition_path=""'
 
@@ -181,13 +179,15 @@ teardown() {
     mkdir -p "$ACQUIS_DIR"
     mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
 
-    rune -124 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
 
     # now, if foo.yaml is empty instead, there won't be valid datasources.
 
     cat /dev/null >"$ACQUIS_DIR"/foo.yaml
 
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -1 wait-for "${CROWDSEC}"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
@@ -212,9 +212,10 @@ teardown() {
 	  type: syslog
 	EOT
 
-    rune -124 timeout 2s env PATH='' "${CROWDSEC}"
     #shellcheck disable=SC2016
-    assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
+    rune -0 wait-for \
+        --err 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in ' \
+        env PATH='' "${CROWDSEC}"
 
     # if all datasources are disabled, crowdsec should exit
 
@@ -222,7 +223,7 @@ teardown() {
     rm -f "$ACQUIS_YAML"
     config_set '.crowdsec_service.acquisition_path=""'
 
-    rune -1 timeout 2s env PATH='' "${CROWDSEC}"
+    rune -1 wait-for env PATH='' "${CROWDSEC}"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 

test/bats/01_cscli.bats

@@ -110,6 +110,37 @@ teardown() {
     assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]'
 }
 
+@test "cscli - required configuration paths" {
+    config=$(cat "${CONFIG_YAML}")
+    configdir=$(config_get '.config_paths.config_dir')
+
+    # required configuration paths with no defaults
+
+    config_set 'del(.config_paths)'
+    rune -1 cscli hub list
+    assert_stderr --partial 'no configuration paths provided'
+    echo "$config" > "${CONFIG_YAML}"
+
+    config_set 'del(.config_paths.data_dir)'
+    rune -1 cscli hub list
+    assert_stderr --partial "please provide a data directory with the 'data_dir' directive in the 'config_paths' section"
+    echo "$config" > "${CONFIG_YAML}"
+
+    # defaults
+
+    config_set 'del(.config_paths.hub_dir)'
+    rune -0 cscli hub list
+    rune -0 cscli config show --key Config.ConfigPaths.HubDir
+    assert_output "$configdir/hub"
+    echo "$config" > "${CONFIG_YAML}"
+
+    config_set 'del(.config_paths.index_path)'
+    rune -0 cscli hub list
+    rune -0 cscli config show --key Config.ConfigPaths.HubIndexFile
+    assert_output "$configdir/hub/.index.json"
+    echo "$config" > "${CONFIG_YAML}"
+}
+
 @test "cscli config show-yaml" {
     rune -0 cscli config show-yaml
     rune -0 yq .common.log_level <(output)
@@ -245,50 +276,23 @@ teardown() {
     assert_output --partial "# bash completion for cscli"
 }
 
-@test "cscli hub list" {
-    # we check for the presence of some objects. There may be others when we
-    # use $PACKAGE_TESTING, so the order is not important.
-
-    rune -0 cscli hub list -o human
-    assert_line --regexp '^ crowdsecurity/linux'
-    assert_line --regexp '^ crowdsecurity/sshd'
-    assert_line --regexp '^ crowdsecurity/dateparse-enrich'
-    assert_line --regexp '^ crowdsecurity/geoip-enrich'
-    assert_line --regexp '^ crowdsecurity/sshd-logs'
-    assert_line --regexp '^ crowdsecurity/syslog-logs'
-    assert_line --regexp '^ crowdsecurity/ssh-bf'
-    assert_line --regexp '^ crowdsecurity/ssh-slow-bf'
-
-    rune -0 cscli hub list -o raw
-    assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$'
-    assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$'
-    assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$'
-    assert_line --regexp '^crowdsecurity/geoip-enrich,enabled,[0-9]+\.[0-9]+,"Populate event with geoloc info : as, country, coords, source range.",parsers$'
-    assert_line --regexp '^crowdsecurity/sshd-logs,enabled,[0-9]+\.[0-9]+,Parse openSSH logs,parsers$'
-    assert_line --regexp '^crowdsecurity/syslog-logs,enabled,[0-9]+\.[0-9]+,,parsers$'
-    assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$'
-    assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$'
-
-    rune -0 cscli hub list -o json
-    rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name' <(output)
-    assert_line 'crowdsecurity/linux'
-    assert_line 'crowdsecurity/sshd'
-    assert_line 'crowdsecurity/dateparse-enrich'
-    assert_line 'crowdsecurity/geoip-enrich'
-    assert_line 'crowdsecurity/sshd-logs'
-    assert_line 'crowdsecurity/syslog-logs'
-    assert_line 'crowdsecurity/ssh-bf'
-    assert_line 'crowdsecurity/ssh-slow-bf'
-}
-
 @test "cscli support dump (smoke test)" {
     rune -0 cscli support dump -f "$BATS_TEST_TMPDIR"/dump.zip
     assert_file_exists "$BATS_TEST_TMPDIR"/dump.zip
 }
 
 @test "cscli explain" {
-    rune -0 cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog --crowdsec "$CROWDSEC"
+    line="Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4"
+
+    rune -0 cscli parsers install crowdsecurity/syslog-logs
+    rune -0 cscli collections install crowdsecurity/sshd
+
+    rune -0 cscli explain --log "$line" --type syslog --only-successful-parsers --crowdsec "$CROWDSEC"
     assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt
+
+    rune -0 cscli parsers remove --all --purge
+    rune -1 cscli explain --log "$line" --type syslog --crowdsec "$CROWDSEC"
+    assert_stderr --partial "unable to load parser dump result: no parser found. Please install the appropriate parser and retry"
 }
 
 @test 'Allow variable expansion and literal $ characters in passwords' {

test/bats/02_nolapi.bats

@@ -24,21 +24,23 @@ teardown() {
 #----------
 
 @test "test without -no-api flag" {
-    rune -124 timeout 2s "${CROWDSEC}"
-    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+    config_set '.common.log_media="stdout"'
+    rune -0 wait-for \
+        --err "CrowdSec Local API listening" \
+        "${CROWDSEC}"
 }
 
 @test "crowdsec should not run without LAPI (-no-api flag)" {
-    # really needs 4 secs on slow boxes
-    rune -1 timeout 4s "${CROWDSEC}" -no-api
+    config_set '.common.log_media="stdout"'
+    rune -1 wait-for "${CROWDSEC}" -no-api
 }
 
 @test "crowdsec should not run without LAPI (no api.server in configuration file)" {
     config_disable_lapi
     config_log_stderr
-    # really needs 4 secs on slow boxes
-    rune -1 timeout 4s "${CROWDSEC}"
-    assert_stderr --partial "crowdsec local API is disabled"
+    rune -0 wait-for \
+        --err "crowdsec local API is disabled" \
+        "${CROWDSEC}"
 }
 
 @test "capi status shouldn't be ok without api.server" {

test/bats/03_noagent.bats

@@ -23,20 +23,25 @@ teardown() {
 #----------
 
 @test "with agent: test without -no-cs flag" {
-    rune -124 timeout 2s "${CROWDSEC}"
-    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+    config_set '.common.log_media="stdout"'
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
 }
 
 @test "no agent: crowdsec LAPI should run (-no-cs flag)" {
-    rune -124 timeout 2s "${CROWDSEC}" -no-cs
+    config_set '.common.log_media="stdout"'
+    rune -0 wait-for \
+        --err "CrowdSec Local API listening" \
+        "${CROWDSEC}" -no-cs
 }
 
 @test "no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" {
     config_disable_agent
     config_log_stderr
-    rune -124 timeout 2s "${CROWDSEC}"
-
-    assert_stderr --partial "crowdsec agent is disabled"
+    rune -0 wait-for \
+        --err "crowdsec agent is disabled" \
+        "${CROWDSEC}"
 }
 
 @test "no agent: cscli config show" {

test/bats/04_capi.bats

@@ -22,6 +22,10 @@ setup() {
 @test "cscli capi status" {
     config_enable_capi
     rune -0 cscli capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX
+    rune -1 cscli capi status
+    assert_stderr --partial "no scenarios installed, abort"
+
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
     rune -0 cscli capi status
     assert_stderr --partial "Loaded credentials from"
     assert_stderr --partial "Trying to authenticate with username"