merge hub-1.5.6 branch

cmd/crowdsec-cli/collections.go

@@ -12,11 +12,15 @@ import (
 )
 
 func NewCollectionsCmd() *cobra.Command {
-	var cmdCollections = &cobra.Command{
-		Use:   "collections [action]",
-		Short: "Manage collections from hub",
-		Long:  `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
-		/*TBD fix help*/
+	cmdCollections := &cobra.Command{
+		Use:   "collections <action> [collection]...",
+		Short: "Manage hub collections",
+		Example: `cscli collections list -a
+cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
+`,
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"collection"},
 		DisableAutoGenTag: true,
@@ -35,142 +39,280 @@ func NewCollectionsCmd() *cobra.Command {
 		},
 	}
 
-	var ignoreError bool
+	cmdCollections.AddCommand(NewCollectionsInstallCmd())
+	cmdCollections.AddCommand(NewCollectionsRemoveCmd())
+	cmdCollections.AddCommand(NewCollectionsUpgradeCmd())
+	cmdCollections.AddCommand(NewCollectionsInspectCmd())
+	cmdCollections.AddCommand(NewCollectionsListCmd())
 
-	var cmdCollectionsInstall = &cobra.Command{
-		Use:     "install collection",
-		Short:   "Install given collection(s)",
-		Long:    `Fetch and install given collection(s) from hub`,
-		Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
+	return cmdCollections
+}
+
+func runCollectionsInstall(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	downloadOnly, err := flags.GetBool("download-only")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	ignoreError, err := flags.GetBool("ignore")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		t := cwhub.GetItem(cwhub.COLLECTIONS, name)
+		if t == nil {
+			nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
+			Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
+
+			continue
+		}
+
+		if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, force, downloadOnly); err != nil {
+			if !ignoreError {
+				return fmt.Errorf("error while installing '%s': %w", name, err)
+			}
+			log.Errorf("Error while installing '%s': %s", name, err)
+		}
+	}
+
+	return nil
+}
+
+func NewCollectionsInstallCmd() *cobra.Command {
+	cmdCollectionsInstall := &cobra.Command{
+		Use:               "install <collection>...",
+		Short:             "Install given collection(s)",
+		Long:              `Fetch and install one or more collections from hub`,
+		Example:           `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.COLLECTIONS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
-					Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
+		RunE: runCollectionsInstall,
 	}
-	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
-	cmdCollections.AddCommand(cmdCollectionsInstall)
 
-	var cmdCollectionsRemove = &cobra.Command{
-		Use:               "remove collection",
+	flags := cmdCollectionsInstall.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, "Ignore errors when installing multiple collections")
+
+	return cmdCollectionsInstall
+}
+
+func runCollectionsRemove(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	purge, err := flags.GetBool("purge")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, force)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one collection to remove or '--all'")
+	}
+
+	for _, name := range args {
+		if !force {
+			item := cwhub.GetItem(cwhub.COLLECTIONS, name)
+			if item == nil {
+				// XXX: this should be in GetItem?
+				return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS)
+			}
+			if len(item.BelongsToCollections) > 0 {
+				log.Warningf("%s belongs to other collections: %s", name, item.BelongsToCollections)
+				log.Warningf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection", name)
+				continue
+			}
+		}
+
+		err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, force)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewCollectionsRemoveCmd() *cobra.Command {
+	cmdCollectionsRemove := &cobra.Command{
+		Use:               "remove <collection>...",
 		Short:             "Remove given collection(s)",
-		Long:              `Remove given collection(s) from hub`,
-		Example:           `cscli collections remove crowdsec/xxx crowdsec/xyz`,
+		Long:              `Remove one or more collections`,
+		Example:           `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
 		Aliases:           []string{"delete"},
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one collection to remove or '--all'")
-			}
-
-			for _, name := range args {
-				if !forceAction {
-					item := cwhub.GetItem(cwhub.COLLECTIONS, name)
-					if item == nil {
-						return fmt.Errorf("unable to retrieve: %s", name)
-					}
-					if len(item.BelongsToCollections) > 0 {
-						log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
-						log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
-						continue
-					}
-				}
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
-			}
-			return nil
-		},
+		RunE: runCollectionsRemove,
 	}
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
-	cmdCollections.AddCommand(cmdCollectionsRemove)
 
-	var cmdCollectionsUpgrade = &cobra.Command{
-		Use:               "upgrade collection",
+	flags := cmdCollectionsRemove.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, "Remove all the collections")
+
+	return cmdCollectionsRemove
+}
+
+func runCollectionsUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one collection to upgrade or '--all'")
+	}
+
+	for _, name := range args {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewCollectionsUpgradeCmd() *cobra.Command {
+	cmdCollectionsUpgrade := &cobra.Command{
+		Use:               "upgrade <collection>...",
 		Short:             "Upgrade given collection(s)",
-		Long:              `Fetch and upgrade given collection(s) from hub`,
-		Example:           `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
+		Long:              `Fetch and upgrade one or more collections from the hub`,
+		Example:           `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one collection to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
-				}
-			}
-			return nil
-		},
+		RunE: runCollectionsUpgrade,
 	}
-	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
-	cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-	cmdCollections.AddCommand(cmdCollectionsUpgrade)
 
-	var cmdCollectionsInspect = &cobra.Command{
-		Use:               "inspect collection",
-		Short:             "Inspect given collection",
-		Long:              `Inspect given collection`,
-		Example:           `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
+	flags := cmdCollectionsUpgrade.Flags()
+	flags.BoolP("all", "a", false, "Upgrade all the collections")
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmdCollectionsUpgrade
+}
+
+func runCollectionsInspect(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewCollectionsInspectCmd() *cobra.Command {
+	cmdCollectionsInspect := &cobra.Command{
+		Use:               "inspect <collection>...",
+		Short:             "Inspect given collection(s)",
+		Long:              `Inspect one or more collections`,
+		Example:           `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			for _, name := range args {
-				InspectItem(name, cwhub.COLLECTIONS)
-			}
-		},
+		RunE: runCollectionsInspect,
 	}
-	cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-	cmdCollections.AddCommand(cmdCollectionsInspect)
 
-	var cmdCollectionsList = &cobra.Command{
-		Use:               "list collection [-a]",
-		Short:             "List all collections",
-		Long:              `List all collections`,
-		Example:           `cscli collections list`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
-		},
-	}
-	cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-	cmdCollections.AddCommand(cmdCollectionsList)
+	flags := cmdCollectionsInspect.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
 
-	return cmdCollections
+	return cmdCollectionsInspect
+}
+
+func runCollectionsList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewCollectionsListCmd() *cobra.Command {
+	cmdCollectionsList := &cobra.Command{
+		Use:   "list [collection... | -a]",
+		Short: "List collections",
+		Long:  `List of installed/available/specified collections`,
+		Example: `cscli collections list
+cscli collections list -a
+cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`,
+		DisableAutoGenTag: true,
+		RunE:              runCollectionsList,
+	}
+
+	flags := cmdCollectionsList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmdCollectionsList
 }

cmd/crowdsec-cli/config_backup.go

@@ -44,7 +44,7 @@ func backupHub(dirPath string) error {
 			//for the local/tainted ones, we backup the full file
 			if v.Tainted || v.Local || !v.UpToDate {
 				//we need to backup stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
 					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
 					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
 						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)

cmd/crowdsec-cli/config_restore.go

@@ -27,10 +27,7 @@ func silentInstallItem(name string, obtype string) (string, error) {
 	if item == nil {
 		return "", fmt.Errorf("error retrieving item")
 	}
-	if downloadOnly && item.Downloaded && item.UpToDate {
-		return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
-	}
-	err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
+	err := cwhub.DownloadLatest(csConfig.Hub, item, false, false)
 	if err != nil {
 		return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
 	}
@@ -38,9 +35,6 @@ func silentInstallItem(name string, obtype string) (string, error) {
 		return "", err
 	}
 
-	if downloadOnly {
-		return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
-	}
 	err = cwhub.EnableItem(csConfig.Hub, item)
 	if err != nil {
 		return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
@@ -54,10 +48,6 @@ func silentInstallItem(name string, obtype string) (string, error) {
 func restoreHub(dirPath string) error {
 	var err error
 
-	if err := csConfig.LoadHub(); err != nil {
-		return err
-	}
-
 	cwhub.SetHubBranch()
 
 	for _, itype := range cwhub.ItemTypes {
@@ -98,7 +88,7 @@ func restoreHub(dirPath string) error {
 			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
 				continue
 			}
-			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
+			if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
 				//we expect a stage here
 				if !file.IsDir() {
 					continue

cmd/crowdsec-cli/hub.go

@@ -15,17 +15,14 @@ import (
 func NewHubCmd() *cobra.Command {
 	var cmdHub = &cobra.Command{
 		Use:   "hub [action]",
-		Short: "Manage Hub",
-		Long: `
-Hub management
+		Short: "Manage hub index",
+		Long: `Hub management
 
 List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
-The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
-		`,
-		Example: `
-cscli hub list   # List all installed configurations
-cscli hub update # Download list of available configurations from the hub
-		`,
+The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
+		Example: `cscli hub list
+cscli hub update
+cscli hub upgrade`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
@@ -45,39 +42,76 @@ cscli hub update # Download list of available configurations from the hub
 	return cmdHub
 }
 
+func runHubList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if err = require.Hub(csConfig); err != nil {
+		return err
+	}
+
+	// use LocalSync to get warnings about tainted / outdated items
+	warn, _ := cwhub.LocalSync(csConfig.Hub)
+	for _, v := range warn {
+		log.Info(v)
+	}
+
+	cwhub.DisplaySummary()
+
+	err = ListItems(color.Output, []string{
+		cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS,
+	}, nil, true, false, all)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func NewHubListCmd() *cobra.Command {
 	var cmdHubList = &cobra.Command{
 		Use:               "list [-a]",
-		Short:             "List installed configs",
+		Short:             "List all installed configurations",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			// use LocalSync to get warnings about tainted / outdated items
-			warn, _ := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
-			}
-			cwhub.DisplaySummary()
-			ListItems(color.Output, []string{
-				cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
-			}, args, true, false, all)
-
-			return nil
-		},
+		RunE: 	    runHubList,
 	}
-	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+
+	flags := cmdHubList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
 
 	return cmdHubList
 }
 
+func runHubUpdate(cmd *cobra.Command, args []string) error {
+	if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
+		if !errors.Is(err, cwhub.ErrIndexNotFound) {
+			return fmt.Errorf("failed to get Hub index : %w", err)
+		}
+		log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
+		cwhub.HubBranch = "master"
+		if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
+			return fmt.Errorf("failed to get Hub index after retry: %w", err)
+		}
+	}
+
+	// use LocalSync to get warnings about tainted / outdated items
+	warn, _ := cwhub.LocalSync(csConfig.Hub)
+	for _, v := range warn {
+		log.Info(v)
+	}
+
+	return nil
+}
+
 func NewHubUpdateCmd() *cobra.Command {
 	var cmdHubUpdate = &cobra.Command{
 		Use:   "update",
-		Short: "Fetch available configs from hub",
+		Short: "Download the latest index (catalog of available configurations)",
 		Long: `
 Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
 `,
@@ -92,37 +126,51 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				return err
-			}
-			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-				if !errors.Is(err, cwhub.ErrIndexNotFound) {
-					return fmt.Errorf("failed to get Hub index : %w", err)
-				}
-				log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
-				cwhub.HubBranch = "master"
-				if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-					return fmt.Errorf("failed to get Hub index after retry: %w", err)
-				}
-			}
-			// use LocalSync to get warnings about tainted / outdated items
-			warn, _ := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
-			}
-
-			return nil
-		},
+		RunE: runHubUpdate,
 	}
 
 	return cmdHubUpdate
 }
 
+func runHubUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	if err := require.Hub(csConfig); err != nil {
+		return err
+	}
+
+	log.Infof("Upgrading collections")
+	if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil {
+		return err
+	}
+
+	log.Infof("Upgrading parsers")
+	if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil {
+		return err
+	}
+
+	log.Infof("Upgrading scenarios")
+	if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil {
+		return err
+	}
+
+	log.Infof("Upgrading postoverflows")
+	if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func NewHubUpgradeCmd() *cobra.Command {
 	var cmdHubUpgrade = &cobra.Command{
 		Use:   "upgrade",
-		Short: "Upgrade all configs installed from hub",
+		Short: "Upgrade all configurations to their latest version",
 		Long: `
 Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
 `,
@@ -137,24 +185,11 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			log.Infof("Upgrading collections")
-			cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			log.Infof("Upgrading parsers")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			log.Infof("Upgrading scenarios")
-			cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			log.Infof("Upgrading postoverflows")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-
-			return nil
-		},
+		RunE: runHubUpgrade,
 	}
-	cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	flags := cmdHubUpgrade.Flags()
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
 
 	return cmdHubUpgrade
 }

cmd/crowdsec-cli/item_metrics.go

@@ -0,0 +1,247 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/trace"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func ShowMetrics(hubItem *cwhub.Item) {
+	switch hubItem.Type {
+	case cwhub.PARSERS:
+		metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		parserMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.SCENARIOS:
+		metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.COLLECTIONS:
+		for _, item := range hubItem.Parsers {
+			metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, item)
+			parserMetricsTable(color.Output, item, metrics)
+		}
+		for _, item := range hubItem.Scenarios {
+			metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, item)
+			scenarioMetricsTable(color.Output, item, metrics)
+		}
+		for _, item := range hubItem.Collections {
+			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
+			if hubItem == nil {
+				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
+			}
+			ShowMetrics(hubItem)
+		}
+	case cwhub.WAAP_RULES:
+		log.Fatalf("FIXME: not implemented yet")
+	default:
+		log.Errorf("item of type '%s' is unknown", hubItem.Type)
+	}
+}
+
+// GetParserMetric is a complete rip from prom2json
+func GetParserMetric(url string, itemName string) map[string]map[string]int {
+	stats := make(map[string]map[string]int)
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric %v", metric.Labels)
+			} else {
+				if srctype, ok := metric.Labels["type"]; ok {
+					source = srctype + ":" + source
+				}
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_reader_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+					stats[source]["parsed"] = 0
+					stats[source]["reads"] = 0
+					stats[source]["unparsed"] = 0
+					stats[source]["hits"] = 0
+				}
+				stats[source]["reads"] += ival
+			case "cs_parser_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			case "cs_node_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["hits"] += ival
+			case "cs_node_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_node_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetScenarioMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["instantiation"] = 0
+	stats["curr_count"] = 0
+	stats["overflow"] = 0
+	stats["pour"] = 0
+	stats["underflow"] = 0
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_bucket_created_total":
+				stats["instantiation"] += ival
+			case "cs_buckets":
+				stats["curr_count"] += ival
+			case "cs_bucket_overflowed_total":
+				stats["overflow"] += ival
+			case "cs_bucket_poured_total":
+				stats["pour"] += ival
+			case "cs_bucket_underflowed_total":
+				stats["underflow"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetPrometheusMetric(url string) []*prom2json.Family {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+
+	return result
+}
+
+type unit struct {
+	value  int64
+	symbol string
+}
+
+var ranges = []unit{
+	{value: 1e18, symbol: "E"},
+	{value: 1e15, symbol: "P"},
+	{value: 1e12, symbol: "T"},
+	{value: 1e9, symbol: "G"},
+	{value: 1e6, symbol: "M"},
+	{value: 1e3, symbol: "k"},
+	{value: 1, symbol: ""},
+}
+
+func formatNumber(num int) string {
+	goodUnit := unit{}
+	for _, u := range ranges {
+		if int64(num) >= u.value {
+			goodUnit = u
+			break
+		}
+	}
+
+	if goodUnit.value == 1 {
+		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
+	}
+
+	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
+	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
+}

cmd/crowdsec-cli/item_suggest.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/agext/levenshtein"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"slices"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+const MaxDistance = 7
+
+func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
+	errMsg := ""
+	if score < MaxDistance {
+		errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem)
+	} else {
+		errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType)
+	}
+	if ignoreErr {
+		log.Error(errMsg)
+	} else {
+		log.Fatalf(errMsg)
+	}
+}
+
+func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
+	allItems := make([]string, 0)
+	nearestScore := 100
+	nearestItem := &cwhub.Item{}
+	hubItems := cwhub.GetItemMap(itemType)
+	for _, item := range hubItems {
+		allItems = append(allItems, item.Name)
+	}
+
+	for _, s := range allItems {
+		d := levenshtein.Distance(itemName, s, nil)
+		if d < nearestScore {
+			nearestScore = d
+			nearestItem = cwhub.GetItem(itemType, s)
+		}
+	}
+	return nearestItem, nearestScore
+}
+
+func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	if err := require.Hub(csConfig); err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+	hubItems := cwhub.GetItemMap(itemType)
+	for _, item := range hubItems {
+		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
+			comp = append(comp, item.Name)
+		}
+	}
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}
+
+func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	if err := require.Hub(csConfig); err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	items, err := cwhub.GetInstalledItemsAsString(itemType)
+	if err != nil {
+		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	if toComplete != "" {
+		for _, item := range items {
+			if strings.Contains(item, toComplete) {
+				comp = append(comp, item)
+			}
+		}
+	} else {
+		comp = items
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}

cmd/crowdsec-cli/items.go

@@ -0,0 +1,174 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"slices"
+	"sort"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+
+func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) {
+	itemNames := cwhub.GetItemNames(itemType)
+
+	notExist := []string{}
+	if len(args) > 0 {
+		installedOnly = false
+		for _, arg := range args {
+			if !slices.Contains(itemNames, arg) {
+				notExist = append(notExist, arg)
+			}
+		}
+	}
+
+	if len(notExist) > 0 {
+		return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
+	}
+
+	if len(args) > 0 {
+		itemNames = args
+	}
+
+	if installedOnly {
+		installed := []string{}
+		for _, item := range itemNames {
+			if cwhub.GetItem(itemType, item).Installed {
+				installed = append(installed, item)
+			}
+		}
+		return installed, nil
+	}
+	return itemNames, nil
+}
+
+
+func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
+	var err error
+	items := make(map[string][]string)
+	for _, itemType := range itemTypes {
+		if items[itemType], err = selectItems(itemType, args, !all); err != nil {
+			return err
+		}
+	}
+		
+	if csConfig.Cscli.Output == "human" {
+		for _, itemType := range itemTypes {
+			listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType])
+		}
+	} else if csConfig.Cscli.Output == "json" {
+		type itemHubStatus struct {
+			Name         string `json:"name"`
+			LocalVersion string `json:"local_version"`
+			LocalPath    string `json:"local_path"`
+			Description  string `json:"description"`
+			UTF8Status   string `json:"utf8_status"`
+			Status       string `json:"status"`
+		}
+
+		hubStatus := make(map[string][]itemHubStatus)
+		for _, itemType := range itemTypes {
+			// empty slice in case there are no items of this type
+			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
+			for i, itemName := range items[itemType] {
+				item := cwhub.GetItem(itemType, itemName)
+				status, emo := item.Status()
+				hubStatus[itemType][i] = itemHubStatus{
+					Name:         item.Name,
+					LocalVersion: item.LocalVersion,
+					LocalPath:    item.LocalPath,
+					Description:  item.Description,
+					Status:       status,
+					UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
+				}
+			}
+			h := hubStatus[itemType]
+			sort.Slice(h, func(i, j int) bool { return h[i].Name < h[j].Name })
+		}
+		x, err := json.MarshalIndent(hubStatus, "", " ")
+		if err != nil {
+			log.Fatalf("failed to unmarshal")
+		}
+		out.Write(x)
+	} else if csConfig.Cscli.Output == "raw" {
+		csvwriter := csv.NewWriter(out)
+		if showHeader {
+			header := []string{"name", "status", "version", "description"}
+			if showType {
+				header = append(header, "type")
+			}
+			err := csvwriter.Write(header)
+			if err != nil {
+				log.Fatalf("failed to write header: %s", err)
+			}
+
+		}
+		for _, itemType := range itemTypes {
+			for _, itemName := range items[itemType] {
+				item := cwhub.GetItem(itemType, itemName)
+				status, _ := item.Status()
+				if item.LocalVersion == "" {
+					item.LocalVersion = "n/a"
+				}
+				row := []string{
+					item.Name,
+					status,
+					item.LocalVersion,
+					item.Description,
+				}
+				if showType {
+					row = append(row, itemType)
+				}
+				err := csvwriter.Write(row)
+				if err != nil {
+					log.Fatalf("failed to write raw output : %s", err)
+				}
+			}
+		}
+		csvwriter.Flush()
+	}
+	return nil
+}
+
+func InspectItem(name string, itemType string, noMetrics bool) error {
+	hubItem := cwhub.GetItem(itemType, name)
+	if hubItem == nil {
+		return fmt.Errorf("can't find '%s' in %s", name, itemType)
+	}
+
+	var (
+		b   []byte
+		err error
+	)
+
+	switch csConfig.Cscli.Output {
+	case "human", "raw":
+		b, err = yaml.Marshal(*hubItem)
+		if err != nil {
+			return fmt.Errorf("unable to marshal item: %s", err)
+		}
+	case "json":
+		b, err = json.MarshalIndent(*hubItem, "", " ")
+		if err != nil {
+			return fmt.Errorf("unable to marshal item: %s", err)
+		}
+	}
+
+	fmt.Printf("%s", string(b))
+
+	if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
+		return nil
+	}
+
+	fmt.Printf("\nCurrent metrics: \n")
+	ShowMetrics(hubItem)
+
+	return nil
+}

cmd/crowdsec-cli/main.go

@@ -29,13 +29,6 @@ var dbClient *database.Client
 var OutputFormat string
 var OutputColor string
 
-var downloadOnly bool
-var forceAction bool
-var purge bool
-var all bool
-
-var prometheusURL string
-
 var mergedConfig string
 
 func initConfig() {
@@ -58,10 +51,8 @@ func initConfig() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		if err := csConfig.LoadCSCLI(); err != nil {
-			log.Fatal(err)
-		}
 	} else {
+		// XXX: check all the defaults
 		csConfig = csconfig.NewDefaultConfig()
 	}
 
@@ -255,7 +246,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewHubTestCmd())
 	rootCmd.AddCommand(NewNotificationsCmd())
 	rootCmd.AddCommand(NewSupportCmd())
-	rootCmd.AddCommand(NewWafRulesCmd())
+	rootCmd.AddCommand(NewWaapRulesCmd())
 
 	if fflag.CscliSetup.IsEnabled() {
 		rootCmd.AddCommand(NewSetupCmd())

cmd/crowdsec-cli/metrics.go

@@ -284,8 +284,20 @@ var noUnit bool
 
 
 func runMetrics(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadPrometheus(); err != nil {
-		return fmt.Errorf("failed to load prometheus config: %w", err)
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noUnit, err = flags.GetBool("no-unit")
+	if err != nil {
+		return err
 	}
 
 	if csConfig.Prometheus == nil {
@@ -296,17 +308,8 @@ func runMetrics(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("prometheus is not enabled, can't show metrics")
 	}
 
-	if prometheusURL == "" {
-		prometheusURL = csConfig.Cscli.PrometheusUrl
-	}
-
-	if prometheusURL == "" {
-		return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
-	}
-
-	err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
-	if err != nil {
-		return fmt.Errorf("could not fetch prometheus metrics: %w", err)
+	if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
+		return err
 	}
 	return nil
 }
@@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: runMetrics,
 	}
-	cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
-	cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
+
+	flags := cmdMetrics.PersistentFlags()
+	flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
+	flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
 
 	return cmdMetrics
 }

cmd/crowdsec-cli/parsers.go

@@ -12,14 +12,14 @@ import (
 )
 
 func NewParsersCmd() *cobra.Command {
-	var cmdParsers = &cobra.Command{
-		Use:   "parsers [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
-		Example: `cscli parsers install crowdsecurity/sshd-logs
-cscli parsers inspect crowdsecurity/sshd-logs
-cscli parsers upgrade crowdsecurity/sshd-logs
-cscli parsers list
-cscli parsers remove crowdsecurity/sshd-logs
+	cmdParsers := &cobra.Command{
+		Use:   "parsers <action> [parser]...",
+		Short: "Manage hub parsers",
+		Example: `cscli parsers list -a
+cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
 `,
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"parser"},
@@ -48,147 +48,258 @@ cscli parsers remove crowdsecurity/sshd-logs
 	return cmdParsers
 }
 
-func NewParsersInstallCmd() *cobra.Command {
-	var ignoreError bool
+func runParsersInstall(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
 
-	var cmdParsersInstall = &cobra.Command{
-		Use:               "install [config]",
+	downloadOnly, err := flags.GetBool("download-only")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	ignoreError, err := flags.GetBool("ignore")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		t := cwhub.GetItem(cwhub.PARSERS, name)
+		if t == nil {
+			nearestItem, score := GetDistance(cwhub.PARSERS, name)
+			Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
+
+			continue
+		}
+
+		if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, force, downloadOnly); err != nil {
+			if !ignoreError {
+				return fmt.Errorf("error while installing '%s': %w", name, err)
+			}
+			log.Errorf("Error while installing '%s': %s", name, err)
+		}
+	}
+
+	return nil
+}
+
+func NewParsersInstallCmd() *cobra.Command {
+	cmdParsersInstall := &cobra.Command{
+		Use:               "install <parser>...",
 		Short:             "Install given parser(s)",
-		Long:              `Fetch and install given parser(s) from hub`,
-		Example:           `cscli parsers install crowdsec/xxx crowdsec/xyz`,
+		Long:              `Fetch and install one or more parsers from the hub`,
+		Example:           `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.PARSERS, args, toComplete)
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS, name)
-					Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
+		RunE: runParsersInstall,
 	}
 
-	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
+	flags := cmdParsersInstall.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, "Ignore errors when installing multiple parsers")
 
 	return cmdParsersInstall
 }
 
+func runParsersRemove(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	purge, err := flags.GetBool("purge")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, force)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one parser to remove or '--all'")
+	}
+
+	for _, name := range args {
+		err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, force)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewParsersRemoveCmd() *cobra.Command {
 	cmdParsersRemove := &cobra.Command{
-		Use:               "remove [config]",
+		Use:               "remove <parser>...",
 		Short:             "Remove given parser(s)",
-		Long:              `Remove given parse(s) from hub`,
-		Example:           `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
+		Long:              `Remove one or more parsers`,
+		Example:           `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
 		Aliases:           []string{"delete"},
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS, args, toComplete)
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one parser to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
-			}
-
-			return nil
-		},
+		RunE: runParsersRemove,
 	}
 
-	cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
+	flags := cmdParsersRemove.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, "Remove all the parsers")
 
 	return cmdParsersRemove
 }
 
+func runParsersUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one parser to upgrade or '--all'")
+	}
+
+	for _, name := range args {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewParsersUpgradeCmd() *cobra.Command {
 	cmdParsersUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
+		Use:               "upgrade <parser>...",
 		Short:             "Upgrade given parser(s)",
-		Long:              `Fetch and upgrade given parser(s) from hub`,
-		Example:           `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
+		Long:              `Fetch and upgrade one or more parsers from the hub`,
+		Example:           `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS, args, toComplete)
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one parser to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
-				}
-			}
-			return nil
-		},
+		RunE: runParsersUpgrade,
 	}
 
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+	flags := cmdParsersUpgrade.Flags()
+	flags.BoolP("all", "a", false, "Upgrade all the parsers")
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
 
 	return cmdParsersUpgrade
 }
 
+func runParsersInspect(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		if err = InspectItem(name, cwhub.PARSERS, noMetrics); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewParsersInspectCmd() *cobra.Command {
-	var cmdParsersInspect = &cobra.Command{
-		Use:               "inspect [name]",
-		Short:             "Inspect given parser",
-		Long:              `Inspect given parser`,
-		Example:           `cscli parsers inspect crowdsec/xxx`,
-		DisableAutoGenTag: true,
+	cmdParsersInspect := &cobra.Command{
+		Use:               "inspect <parser>",
+		Short:             "Inspect a parser",
+		Long:              `Inspect a parser`,
+		Example:           `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
 		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS)
-		},
+		RunE: runParsersInspect,
 	}
 
-	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
+	flags := cmdParsersInspect.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
 
 	return cmdParsersInspect
 }
 
-func NewParsersListCmd() *cobra.Command {
-	var cmdParsersList = &cobra.Command{
-		Use:   "list [name]",
-		Short: "List all parsers or given one",
-		Long:  `List all parsers or given one`,
-		Example: `cscli parsers list
-cscli parser list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
-		},
+func runParsersList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
 	}
 
-	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+	if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewParsersListCmd() *cobra.Command {
+	cmdParsersList := &cobra.Command{
+		Use:   "list [parser... | -a]",
+		Short: "List parsers",
+		Long:  `List of installed/available/specified parsers`,
+		Example: `cscli parsers list
+cscli parsers list -a
+cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		DisableAutoGenTag: true,
+		RunE:              runParsersList,
+	}
+
+	flags := cmdParsersList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
 
 	return cmdParsersList
 }

cmd/crowdsec-cli/postoverflows.go

@@ -13,13 +13,14 @@ import (
 
 func NewPostOverflowsCmd() *cobra.Command {
 	cmdPostOverflows := &cobra.Command{
-		Use:   "postoverflows [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
-		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
-		cscli postoverflows inspect crowdsecurity/cdn-whitelist
-		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
-		cscli postoverflows list
-		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
+		Use:   "postoverflows <action> [postoverflow]...",
+		Short: "Manage hub postoverflows",
+		Example: `cscli postoverflows list -a
+cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
+`,
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"postoverflow"},
 		DisableAutoGenTag: true,
@@ -47,145 +48,259 @@ func NewPostOverflowsCmd() *cobra.Command {
 	return cmdPostOverflows
 }
 
-func NewPostOverflowsInstallCmd() *cobra.Command {
-	var ignoreError bool
+func runPostOverflowsInstall(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
 
+	downloadOnly, err := flags.GetBool("download-only")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	ignoreError, err := flags.GetBool("ignore")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name)
+		if t == nil {
+			nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name)
+			Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError)
+
+			continue
+		}
+
+		if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil {
+			if !ignoreError {
+				return fmt.Errorf("error while installing '%s': %w", name, err)
+			}
+			log.Errorf("Error while installing '%s': %s", name, err)
+		}
+	}
+
+	return nil
+}
+
+func NewPostOverflowsInstallCmd() *cobra.Command {
 	cmdPostOverflowsInstall := &cobra.Command{
-		Use:               "install [config]",
+		Use:               "install <postoverflow>...",
 		Short:             "Install given postoverflow(s)",
-		Long:              `Fetch and install given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
+		Long:              `Fetch and install one or more postoverflows from the hub`,
+		Example:           `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
-					Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
+			return compAllItems(cwhub.POSTOVERFLOWS, args, toComplete)
 		},
+		RunE: runPostOverflowsInstall,
 	}
 
-	cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
+	flags := cmdPostOverflowsInstall.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, "Ignore errors when installing multiple postoverflows")
 
 	return cmdPostOverflowsInstall
 }
 
+func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	purge, err := flags.GetBool("purge")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
+	}
+
+	for _, name := range args {
+		err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewPostOverflowsRemoveCmd() *cobra.Command {
 	cmdPostOverflowsRemove := &cobra.Command{
-		Use:               "remove [config]",
+		Use:               "remove <postoverflow>...",
 		Short:             "Remove given postoverflow(s)",
-		Long:              `remove given postoverflow(s)`,
-		Example:           `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
+		Long:              `remove one or more postoverflows from the hub`,
+		Example:           `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
 		Aliases:           []string{"delete"},
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
-			}
-
-			return nil
+			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
 		},
+		RunE: runPostOverflowsRemove,
 	}
 
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
+	flags := cmdPostOverflowsRemove.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, "Delete all the postoverflows")
 
 	return cmdPostOverflowsRemove
 }
 
-func NewPostOverflowsUpgradeCmd() *cobra.Command {
-	cmdPostOverflowsUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given postoverflow(s)",
-		Long:              `Fetch and Upgrade given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
-				}
-			}
-			return nil
-		},
+func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
 	}
 
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
+	}
+
+	for _, name := range args {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewPostOverflowsUpgradeCmd() *cobra.Command {
+	cmdPostOverflowsUpgrade := &cobra.Command{
+		Use:               "upgrade <postoverflow>...",
+		Short:             "Upgrade given postoverflow(s)",
+		Long:              `Fetch and upgrade one or more postoverflows from the hub`,
+		Example:           `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
+		},
+		RunE: runPostOverflowUpgrade,
+	}
+
+	flags := cmdPostOverflowsUpgrade.Flags()
+	flags.BoolP("all", "a", false, "Upgrade all the postoverflows")
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
 
 	return cmdPostOverflowsUpgrade
 }
 
+func runPostOverflowsInspect(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		if err = InspectItem(name, cwhub.POSTOVERFLOWS, noMetrics); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewPostOverflowsInspectCmd() *cobra.Command {
 	cmdPostOverflowsInspect := &cobra.Command{
-		Use:               "inspect [config]",
-		Short:             "Inspect given postoverflow",
-		Long:              `Inspect given postoverflow`,
-		Example:           `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
+		Use:               "inspect <postoverflow>",
+		Short:             "Inspect a postoverflow",
+		Long:              `Inspect a postoverflow`,
+		Example:           `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
 		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS_OVFLW)
+			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
 		},
+		RunE: runPostOverflowsInspect,
 	}
 
+	flags := cmdPostOverflowsInspect.Flags()
+
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
+
 	return cmdPostOverflowsInspect
 }
 
-func NewPostOverflowsListCmd() *cobra.Command {
-	cmdPostOverflowsList := &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all postoverflows or given one",
-		Long:  `List all postoverflows or given one`,
-		Example: `cscli postoverflows list
-cscli postoverflows list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
-		},
+func runPostOverflowsList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
 	}
 
-	cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+	if err = ListItems(color.Output, []string{cwhub.POSTOVERFLOWS}, args, false, true, all); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewPostOverflowsListCmd() *cobra.Command {
+	cmdPostOverflowsList := &cobra.Command{
+		Use:   "list [postoverflow]...",
+		Short: "List postoverflows",
+		Long:  `List of installed/available/specified postoverflows`,
+		Example: `cscli postoverflows list
+cscli postoverflows list -a
+cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		DisableAutoGenTag: true,
+		RunE:              runPostOverflowsList,
+	}
+
+	flags := cmdPostOverflowsList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
 
 	return cmdPostOverflowsList
 }

cmd/crowdsec-cli/require/require.go

@@ -65,10 +65,6 @@ func Notifications(c *csconfig.Config) error {
 }
 
 func Hub (c *csconfig.Config) error {
-	if err := c.LoadHub(); err != nil {
-		return err
-	}
-
 	if c.Hub == nil {
 		return fmt.Errorf("you must configure cli before interacting with hub")
 	}

cmd/crowdsec-cli/scenarios.go

@@ -12,14 +12,14 @@ import (
 )
 
 func NewScenariosCmd() *cobra.Command {
-	var cmdScenarios = &cobra.Command{
-		Use:   "scenarios [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
-		Example: `cscli scenarios list [-a]
-cscli scenarios install crowdsecurity/ssh-bf
-cscli scenarios inspect crowdsecurity/ssh-bf
-cscli scenarios upgrade crowdsecurity/ssh-bf
-cscli scenarios remove crowdsecurity/ssh-bf
+	cmdScenarios := &cobra.Command{
+		Use:   "scenarios <action> [scenario]...",
+		Short: "Manage hub scenarios",
+		Example: `cscli scenarios list -a
+cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
 `,
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"scenario"},
@@ -48,141 +48,258 @@ cscli scenarios remove crowdsecurity/ssh-bf
 	return cmdScenarios
 }
 
-func NewCmdScenariosInstall() *cobra.Command {
-	var ignoreError bool
+func runScenariosInstall(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
 
-	var cmdScenariosInstall = &cobra.Command{
-		Use:     "install [config]",
-		Short:   "Install given scenario(s)",
-		Long:    `Fetch and install given scenario(s) from hub`,
-		Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
+	downloadOnly, err := flags.GetBool("download-only")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	ignoreError, err := flags.GetBool("ignore")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		t := cwhub.GetItem(cwhub.SCENARIOS, name)
+		if t == nil {
+			nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
+			Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
+
+			continue
+		}
+
+		if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, force, downloadOnly); err != nil {
+			if !ignoreError {
+				return fmt.Errorf("error while installing '%s': %w", name, err)
+			}
+			log.Errorf("Error while installing '%s': %s", name, err)
+		}
+	}
+
+	return nil
+}
+
+func NewCmdScenariosInstall() *cobra.Command {
+	cmdScenariosInstall := &cobra.Command{
+		Use:               "install <scenario>...",
+		Short:             "Install given scenario(s)",
+		Long:              `Fetch and install one or more scenarios from the hub`,
+		Example:           `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.SCENARIOS, args, toComplete)
 		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.SCENARIOS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
-					Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
+		RunE: runScenariosInstall,
 	}
-	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
+
+	flags := cmdScenariosInstall.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, "Ignore errors when installing multiple scenarios")
 
 	return cmdScenariosInstall
 }
 
+func runScenariosRemove(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	purge, err := flags.GetBool("purge")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, force)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one scenario to remove or '--all'")
+	}
+
+	for _, name := range args {
+		err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, force)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewCmdScenariosRemove() *cobra.Command {
-	var cmdScenariosRemove = &cobra.Command{
-		Use:     "remove [config]",
-		Short:   "Remove given scenario(s)",
-		Long:    `remove given scenario(s)`,
-		Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
-		Aliases: []string{"delete"},
+	cmdScenariosRemove := &cobra.Command{
+		Use:               "remove <scenario>...",
+		Short:             "Remove given scenario(s)",
+		Long:              `remove one or more scenarios`,
+		Example:           `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
 		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one scenario to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
-			}
-			return nil
-		},
+		RunE: runScenariosRemove,
 	}
-	cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
+
+	flags := cmdScenariosRemove.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, "Remove all the scenarios")
 
 	return cmdScenariosRemove
 }
 
+func runScenariosUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
+	}
+
+	for _, name := range args {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewCmdScenariosUpgrade() *cobra.Command {
-	var cmdScenariosUpgrade = &cobra.Command{
-		Use:     "upgrade [config]",
-		Short:   "Upgrade given scenario(s)",
-		Long:    `Fetch and Upgrade given scenario(s) from hub`,
-		Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
+	cmdScenariosUpgrade := &cobra.Command{
+		Use:               "upgrade <scenario>...",
+		Short:             "Upgrade given scenario(s)",
+		Long:              `Fetch and upgrade one or more scenarios from the hub`,
+		Example:           `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
 		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
-				}
-			}
-			return nil
-		},
+		RunE: runScenariosUpgrade,
 	}
-	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
-	cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	flags := cmdScenariosUpgrade.Flags()
+	flags.BoolP("all", "a", false, "Upgrade all the scenarios")
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
 
 	return cmdScenariosUpgrade
 }
 
+func runScenariosInspect(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		if err = InspectItem(name, cwhub.SCENARIOS, noMetrics); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func NewCmdScenariosInspect() *cobra.Command {
-	var cmdScenariosInspect = &cobra.Command{
-		Use:     "inspect [config]",
-		Short:   "Inspect given scenario",
-		Long:    `Inspect given scenario`,
-		Example: `cscli scenarios inspect crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
+	cmdScenariosInspect := &cobra.Command{
+		Use:               "inspect <scenario>",
+		Short:             "Inspect a scenario",
+		Long:              `Inspect a scenario`,
+		Example:           `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
 		},
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.SCENARIOS)
-		},
+		RunE: runScenariosInspect,
 	}
-	cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
+
+	flags := cmdScenariosInspect.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
 
 	return cmdScenariosInspect
 }
 
-func NewCmdScenariosList() *cobra.Command {
-	var cmdScenariosList = &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all scenario(s) or given one",
-		Long:  `List all scenario(s) or given one`,
-		Example: `cscli scenarios list
-cscli scenarios list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
-		},
+func runScenariosList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
 	}
-	cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+
+	if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewCmdScenariosList() *cobra.Command {
+	cmdScenariosList := &cobra.Command{
+		Use:   "list [scenario]...",
+		Short: "List scenarios",
+		Long:  `List of installed/available/specified scenarios`,
+		Example: `cscli scenarios list
+cscli scenarios list -a
+cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		DisableAutoGenTag: true,
+		RunE:              runScenariosList,
+	}
+
+	flags := cmdScenariosList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
 
 	return cmdScenariosList
 }

cmd/crowdsec-cli/support.go

@@ -58,10 +58,6 @@ func stripAnsiString(str string) string {
 
 func collectMetrics() ([]byte, []byte, error) {
 	log.Info("Collecting prometheus metrics")
-	err := csConfig.LoadPrometheus()
-	if err != nil {
-		return nil, nil, err
-	}
 
 	if csConfig.Cscli.PrometheusUrl == "" {
 		log.Warn("No Prometheus URL configured, metrics will not be collected")
@@ -69,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) {
 	}
 
 	humanMetrics := bytes.NewBuffer(nil)
-	err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
+	err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
 
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
 	}
 
-	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
+	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
 	}
@@ -135,7 +131,9 @@ func collectOSInfo() ([]byte, error) {
 func collectHubItems(itemType string) []byte {
 	out := bytes.NewBuffer(nil)
 	log.Infof("Collecting %s list", itemType)
-	ListItems(out, []string{itemType}, []string{}, false, true, all)
+	if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
 	return out.Bytes()
 }
 
@@ -335,7 +333,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 			if !skipHub {
 				infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
 				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
-				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
+				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.POSTOVERFLOWS)
 				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
 			}
 

cmd/crowdsec-cli/utils.go

@@ -1,36 +1,17 @@
 package main
 
 import (
-	"encoding/csv"
-	"encoding/json"
 	"fmt"
-	"io"
-	"math"
 	"net"
-	"net/http"
-	"slices"
-	"strconv"
 	"strings"
-	"time"
 
-	"github.com/agext/levenshtein"
-	"github.com/fatih/color"
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-const MaxDistance = 7
-
 func printHelp(cmd *cobra.Command) {
 	err := cmd.Help()
 	if err != nil {
@@ -38,197 +19,6 @@ func printHelp(cmd *cobra.Command) {
 	}
 }
 
-func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
-	errMsg := ""
-	if score < MaxDistance {
-		errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
-	} else {
-		errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
-	}
-	if ignoreErr {
-		log.Error(errMsg)
-	} else {
-		log.Fatalf(errMsg)
-	}
-}
-
-func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
-	allItems := make([]string, 0)
-	nearestScore := 100
-	nearestItem := &cwhub.Item{}
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		allItems = append(allItems, item.Name)
-	}
-
-	for _, s := range allItems {
-		d := levenshtein.Distance(itemName, s, nil)
-		if d < nearestScore {
-			nearestScore = d
-			nearestItem = cwhub.GetItem(itemType, s)
-		}
-	}
-	return nearestItem, nearestScore
-}
-
-func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
-			comp = append(comp, item.Name)
-		}
-	}
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	items, err := cwhub.GetInstalledItemsAsString(itemType)
-	if err != nil {
-		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-
-	if toComplete != "" {
-		for _, item := range items {
-			if strings.Contains(item, toComplete) {
-				comp = append(comp, item)
-			}
-		}
-	} else {
-		comp = items
-	}
-
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
-	var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
-
-	for _, itemType := range itemTypes {
-		itemName := ""
-		if len(args) == 1 {
-			itemName = args[0]
-		}
-		hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
-	}
-
-	if csConfig.Cscli.Output == "human" {
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
-		}
-	} else if csConfig.Cscli.Output == "json" {
-		x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
-		if err != nil {
-			log.Fatalf("failed to unmarshal")
-		}
-		out.Write(x)
-	} else if csConfig.Cscli.Output == "raw" {
-		csvwriter := csv.NewWriter(out)
-		if showHeader {
-			header := []string{"name", "status", "version", "description"}
-			if showType {
-				header = append(header, "type")
-			}
-			err := csvwriter.Write(header)
-			if err != nil {
-				log.Fatalf("failed to write header: %s", err)
-			}
-
-		}
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			for _, status := range statuses {
-				if status.LocalVersion == "" {
-					status.LocalVersion = "n/a"
-				}
-				row := []string{
-					status.Name,
-					status.Status,
-					status.LocalVersion,
-					status.Description,
-				}
-				if showType {
-					row = append(row, itemType)
-				}
-				err := csvwriter.Write(row)
-				if err != nil {
-					log.Fatalf("failed to write raw output : %s", err)
-				}
-			}
-		}
-		csvwriter.Flush()
-	}
-}
-
-func InspectItem(name string, objecitemType string) {
-
-	hubItem := cwhub.GetItem(objecitemType, name)
-	if hubItem == nil {
-		log.Fatalf("unable to retrieve item.")
-	}
-	var b []byte
-	var err error
-	switch csConfig.Cscli.Output {
-	case "human", "raw":
-		b, err = yaml.Marshal(*hubItem)
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	case "json":
-		b, err = json.MarshalIndent(*hubItem, "", " ")
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	}
-	fmt.Printf("%s", string(b))
-	if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
-		return
-	}
-
-	if prometheusURL == "" {
-		//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
-		//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
-		listenAddr := csConfig.Prometheus.ListenAddr
-		if listenAddr == "" {
-			listenAddr = "127.0.0.1"
-		}
-		listenPort := csConfig.Prometheus.ListenPort
-		if listenPort == 0 {
-			listenPort = 6060
-		}
-		prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
-		log.Debugf("No prometheus URL provided using: %s", prometheusURL)
-	}
-
-	fmt.Printf("\nCurrent metrics : \n")
-	ShowMetrics(hubItem)
-}
-
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
 
 	/*if a range is provided, change the scope*/
@@ -259,234 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 	return nil
 }
 
-func ShowMetrics(hubItem *cwhub.Item) {
-	switch hubItem.Type {
-	case cwhub.PARSERS:
-		metrics := GetParserMetric(prometheusURL, hubItem.Name)
-		parserMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.SCENARIOS:
-		metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
-		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.COLLECTIONS:
-		for _, item := range hubItem.Parsers {
-			metrics := GetParserMetric(prometheusURL, item)
-			parserMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Scenarios {
-			metrics := GetScenarioMetric(prometheusURL, item)
-			scenarioMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Collections {
-			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
-			if hubItem == nil {
-				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
-			}
-			ShowMetrics(hubItem)
-		}
-	case cwhub.WAAP_RULES:
-		log.Fatalf("FIXME: not implemented yet")
-	default:
-		log.Errorf("item of type '%s' is unknown", hubItem.Type)
-	}
-}
-
-// GetParserMetric is a complete rip from prom2json
-func GetParserMetric(url string, itemName string) map[string]map[string]int {
-	stats := make(map[string]map[string]int)
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			source, ok := metric.Labels["source"]
-			if !ok {
-				log.Debugf("no source in Metric %v", metric.Labels)
-			} else {
-				if srctype, ok := metric.Labels["type"]; ok {
-					source = srctype + ":" + source
-				}
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_reader_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-					stats[source]["parsed"] = 0
-					stats[source]["reads"] = 0
-					stats[source]["unparsed"] = 0
-					stats[source]["hits"] = 0
-				}
-				stats[source]["reads"] += ival
-			case "cs_parser_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_parser_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			case "cs_node_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["hits"] += ival
-			case "cs_node_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_node_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetScenarioMetric(url string, itemName string) map[string]int {
-	stats := make(map[string]int)
-
-	stats["instantiation"] = 0
-	stats["curr_count"] = 0
-	stats["overflow"] = 0
-	stats["pour"] = 0
-	stats["underflow"] = 0
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_bucket_created_total":
-				stats["instantiation"] += ival
-			case "cs_buckets":
-				stats["curr_count"] += ival
-			case "cs_bucket_overflowed_total":
-				stats["overflow"] += ival
-			case "cs_bucket_poured_total":
-				stats["pour"] += ival
-			case "cs_bucket_underflowed_total":
-				stats["underflow"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetPrometheusMetric(url string) []*prom2json.Family {
-	mfChan := make(chan *dto.MetricFamily, 1024)
-
-	// Start with the DefaultTransport for sane defaults.
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	// Conservatively disable HTTP keep-alives as this program will only
-	// ever need a single HTTP request.
-	transport.DisableKeepAlives = true
-	// Timeout early if the server doesn't even return the headers.
-	transport.ResponseHeaderTimeout = time.Minute
-
-	go func() {
-		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
-		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
-		if err != nil {
-			log.Fatalf("failed to fetch prometheus metrics : %v", err)
-		}
-	}()
-
-	result := []*prom2json.Family{}
-	for mf := range mfChan {
-		result = append(result, prom2json.NewFamily(mf))
-	}
-	log.Debugf("Finished reading prometheus output, %d entries", len(result))
-
-	return result
-}
-
-type unit struct {
-	value  int64
-	symbol string
-}
-
-var ranges = []unit{
-	{value: 1e18, symbol: "E"},
-	{value: 1e15, symbol: "P"},
-	{value: 1e12, symbol: "T"},
-	{value: 1e9, symbol: "G"},
-	{value: 1e6, symbol: "M"},
-	{value: 1e3, symbol: "k"},
-	{value: 1, symbol: ""},
-}
-
-func formatNumber(num int) string {
-	goodUnit := unit{}
-	for _, u := range ranges {
-		if int64(num) >= u.value {
-			goodUnit = u
-			break
-		}
-	}
-
-	if goodUnit.value == 1 {
-		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
-	}
-
-	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
-	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
-}
-
 func getDBClient() (*database.Client, error) {
 	var err error
 	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
@@ -520,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
 	}
 
 	return slice
-
 }

cmd/crowdsec-cli/utils_table.go

@@ -10,14 +10,16 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
+func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) {
 	t := newLightTable(out)
 	t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	for _, status := range statuses {
-		t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
+	for itemName := range itemNames {
+		item := cwhub.GetItem(itemType, itemNames[itemName])
+		status, emo := item.Status()
+		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.LocalVersion, item.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()

cmd/crowdsec-cli/waap_rules.go

@@ -7,36 +7,28 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func NewWafRulesCmd() *cobra.Command {
-	var cmdWafRules = &cobra.Command{
-		Use:   "waap-rules [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect waf-rule(s) from hub",
-		Example: `cscli waap-rules install crowdsecurity/core-rule-set
-cscli waap-rules inspect crowdsecurity/core-rule-set
-cscli waap-rules upgrade crowdsecurity/core-rule-set
-cscli waap-rules list
-cscli waap-rules remove crowdsecurity/core-rule-set
+func NewWaapRulesCmd() *cobra.Command {
+	cmdWaapRules := &cobra.Command{
+		Use:   "waap-rules <action> [waap-rule]...",
+		Short: "Manage hub waap rules",
+		Example: `cscli waap-rules list -a
+cscli waap-rules install crowdsecurity/crs
+cscli waap-rules inspect crowdsecurity/crs
+cscli waap-rules upgrade crowdsecurity/crs
+cscli waap-rules remove crowdsecurity/crs
 `,
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"waap-rule"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
+			if err := require.Hub(csConfig); err != nil {
+				return err
 			}
 
-			cwhub.SetHubBranch()
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -47,148 +39,267 @@ cscli waap-rules remove crowdsecurity/core-rule-set
 		},
 	}
 
-	cmdWafRules.AddCommand(NewWafRulesInstallCmd())
-	cmdWafRules.AddCommand(NewWafRulesRemoveCmd())
-	cmdWafRules.AddCommand(NewWafRulesUpgradeCmd())
-	cmdWafRules.AddCommand(NewWafRulesInspectCmd())
-	cmdWafRules.AddCommand(NewWafRulesListCmd())
+	cmdWaapRules.AddCommand(NewCmdWaapRulesInstall())
+	cmdWaapRules.AddCommand(NewCmdWaapRulesRemove())
+	cmdWaapRules.AddCommand(NewCmdWaapRulesUpgrade())
+	cmdWaapRules.AddCommand(NewCmdWaapRulesInspect())
+	cmdWaapRules.AddCommand(NewCmdWaapRulesList())
 
-	return cmdWafRules
+	return cmdWaapRules
 }
 
-func NewWafRulesInstallCmd() *cobra.Command {
-	var ignoreError bool
+func runWaapRulesInstall(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
 
-	var cmdWafRulesInstall = &cobra.Command{
-		Use:               "install [config]",
-		Short:             "Install given waap-rule(s)",
-		Long:              `Fetch and install given waap-rule(s) from hub`,
-		Example:           `cscli waap-rules install crowdsec/xxx crowdsec/xyz`,
+	downloadOnly, err := flags.GetBool("download-only")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	ignoreError, err := flags.GetBool("ignore")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		t := cwhub.GetItem(cwhub.WAAP_RULES, name)
+		if t == nil {
+			nearestItem, score := GetDistance(cwhub.WAAP_RULES, name)
+			Suggest(cwhub.WAAP_RULES, name, nearestItem.Name, score, ignoreError)
+
+			continue
+		}
+
+		if err := cwhub.InstallItem(csConfig, name, cwhub.WAAP_RULES, force, downloadOnly); err != nil {
+			if !ignoreError {
+				return fmt.Errorf("error while installing '%s': %w", name, err)
+			}
+			log.Errorf("Error while installing '%s': %s", name, err)
+		}
+	}
+
+	return nil
+}
+
+func NewCmdWaapRulesInstall() *cobra.Command {
+	cmdWaapRulesInstall := &cobra.Command{
+		Use:               "install <waap-rule>...",
+		Short:             "Install given waap rule(s)",
+		Long:              `Fetch and install one or more waap rules from the hub`,
+		Example:           `cscli waap-rules install crowdsecurity/crs`,
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.WAAP_RULES, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.WAAP_RULES, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.WAAP_RULES, name)
-					Suggest(cwhub.WAAP_RULES, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.WAAP_RULES, forceAction, downloadOnly); err != nil {
-					if ignoreError {
-						log.Errorf("Error while installing '%s': %s", name, err)
-					} else {
-						log.Fatalf("Error while installing '%s': %s", name, err)
-					}
-				}
-			}
-		},
+		RunE: runWaapRulesInstall,
 	}
-	cmdWafRulesInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdWafRulesInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdWafRulesInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple waf rules")
 
-	return cmdWafRulesInstall
+	flags := cmdWaapRulesInstall.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, "Ignore errors when installing multiple waap rules")
+
+	return cmdWaapRulesInstall
 }
 
-func NewWafRulesRemoveCmd() *cobra.Command {
-	var cmdWafRulesRemove = &cobra.Command{
-		Use:               "remove [config]",
-		Short:             "Remove given waf-rule(s)",
-		Long:              `Remove given waf-rule(s) from hub`,
+func runWaapRulesRemove(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	purge, err := flags.GetBool("purge")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		err := cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, "", all, purge, force)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one waap rule to remove or '--all'")
+	}
+
+	for _, name := range args {
+		err := cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, name, all, purge, force)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewCmdWaapRulesRemove() *cobra.Command {
+	cmdWaapRulesRemove := &cobra.Command{
+		Use:               "remove <waap-rule>...",
+		Short:             "Remove given waap rule(s)",
+		Long:              `remove one or more waap rules`,
+		Example:           `cscli waap-rules remove crowdsecurity/crs`,
 		Aliases:           []string{"delete"},
-		Example:           `cscli waap-rules remove crowdsec/xxx crowdsec/xyz`,
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.WAAP_RULES, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, "", all, purge, forceAction)
-				return
-			}
-
-			if len(args) == 0 {
-				log.Fatalf("Specify at least one waf rule to remove or '--all' flag.")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, name, all, purge, forceAction)
-			}
-		},
+		RunE: runWaapRulesRemove,
 	}
-	cmdWafRulesRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdWafRulesRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdWafRulesRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the waf rules")
 
-	return cmdWafRulesRemove
+	flags := cmdWaapRulesRemove.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, "Remove all the waap rules")
+
+	return cmdWaapRulesRemove
 }
 
-func NewWafRulesUpgradeCmd() *cobra.Command {
-	var cmdWafRulesUpgrade = &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given waf-rule(s)",
-		Long:              `Fetch and upgrade given waf-rule(s) from hub`,
-		Example:           `cscli waap-rules upgrade crowdsec/xxx crowdsec/xyz`,
+func runWaapRulesUpgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if all {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, "", force); err != nil {
+			return err
+		}
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one waap rule to upgrade or '--all'")
+	}
+
+	for _, name := range args {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, name, force); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewCmdWaapRulesUpgrade() *cobra.Command {
+	cmdWaapRulesUpgrade := &cobra.Command{
+		Use:               "upgrade <waap-rule>...",
+		Short:             "Upgrade given waap rule(s)",
+		Long:              `Fetch and upgrade one or more waap rules from the hub`,
+		Example:           `cscli waap-rules upgrade crowdsecurity/crs`,
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.WAAP_RULES, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					log.Fatalf("no target waf rule to upgrade")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, name, forceAction)
-				}
-			}
-		},
+		RunE: runWaapRulesUpgrade,
 	}
-	cmdWafRulesUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the waf rules")
-	cmdWafRulesUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
 
-	return cmdWafRulesUpgrade
+	flags := cmdWaapRulesUpgrade.Flags()
+	flags.BoolP("all", "a", false, "Upgrade all the waap rules")
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmdWaapRulesUpgrade
 }
 
-func NewWafRulesInspectCmd() *cobra.Command {
-	var cmdWafRulesInspect = &cobra.Command{
-		Use:               "inspect [name]",
-		Short:             "Inspect given waf rule",
-		Long:              `Inspect given waf rule`,
-		Example:           `cscli waap-rules inspect crowdsec/xxx`,
-		DisableAutoGenTag: true,
+func runWaapRulesInspect(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		if err = InspectItem(name, cwhub.WAAP_RULES, noMetrics); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func NewCmdWaapRulesInspect() *cobra.Command {
+	cmdWaapRulesInspect := &cobra.Command{
+		Use:               "inspect <waap-rule>",
+		Short:             "Inspect a waap rule",
+		Long:              `Inspect a waap rule`,
+		Example:           `cscli waap-rules inspect crowdsecurity/crs`,
 		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.WAAP_RULES, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.WAAP_RULES)
-		},
+		RunE: runWaapRulesInspect,
 	}
-	cmdWafRulesInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
 
-	return cmdWafRulesInspect
+	flags := cmdWaapRulesInspect.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
+
+	return cmdWaapRulesInspect
 }
 
-func NewWafRulesListCmd() *cobra.Command {
-	var cmdWafRulesList = &cobra.Command{
-		Use:   "list [name]",
-		Short: "List all waf rules or given one",
-		Long:  `List all waf rules or given one`,
+func runWaapRulesList(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	if err = ListItems(color.Output, []string{cwhub.WAAP_RULES}, args, false, true, all); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NewCmdWaapRulesList() *cobra.Command {
+	cmdWaapRulesList := &cobra.Command{
+		Use:   "list [waap-rule]...",
+		Short: "List waap rules",
+		Long:  `List of installed/available/specified waap rules`,
 		Example: `cscli waap-rules list
-cscli waap-rules list crowdsecurity/xxx`,
+cscli waap-rules list -a
+cscli waap-rules list crowdsecurity/crs`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.WAAP_RULES}, args, false, true, all)
-		},
+		RunE:              runWaapRulesList,
 	}
-	cmdWafRulesList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
 
-	return cmdWafRulesList
+	flags := cmdWaapRulesList.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmdWaapRulesList
 }

cmd/crowdsec/main.go

@@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
 	cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
 	if err != nil {
-		return nil, err
-	}
-
-	if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
-		return nil, fmt.Errorf("unable to load configuration: common section is empty")
+		return nil, fmt.Errorf("while loading configuration file: %w", err)
 	}
 
 	cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		dumpStates = true
 	}
 
-	// Configuration paths are dependency to load crowdsec configuration
-	if err := cConfig.LoadConfigurationPaths(); err != nil {
-		return nil, err
-	}
-
 	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
 		// if we're in time-machine mode, we don't want to log to file
 		cConfig.Common.LogMedia = "stdout"

cmd/crowdsec/metrics.go

@@ -152,14 +152,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 	if !config.Enabled {
 		return
 	}
-	if config.ListenAddr == "" {
-		log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
-		config.ListenAddr = "127.0.0.1"
-	}
-	if config.ListenPort == 0 {
-		log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
-		config.ListenPort = 6060
-	}
 
 	// Registering prometheus
 	// If in aggregated mode, do not register events associated with a source, to keep the cardinality low

config/config.yaml

@@ -6,7 +6,6 @@ common:
   log_max_size: 20
   compress_logs: true
   log_max_files: 10
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

config/config_win.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/config_win_no_lapi.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/dev.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: true
   log_media: stdout
   log_level: info
-  working_dir: .
 config_paths:
   config_dir: ./config
   data_dir: ./data/   

config/user.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data

docker/config.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

pkg/csconfig/api.go

@@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
 		log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
 	}
 
-	if err := c.LoadCommon(); err != nil {
-		return fmt.Errorf("loading common configuration: %s", err)
-	}
-
 	c.API.Server.LogDir = c.Common.LogDir
 	c.API.Server.LogMedia = c.Common.LogMedia
 	c.API.Server.CompressLogs = c.Common.CompressLogs

pkg/csconfig/api_test.go

@@ -3,7 +3,6 @@ package csconfig
 import (
 	"net"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) {
 	err := tmpLAPI.LoadProfiles()
 	require.NoError(t, err)
 
-	LogDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
 	logLevel := log.InfoLevel
 	config := &Config{}
 	fcontent, err := os.ReadFile("./testdata/config.yaml")
@@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) {
 					DbPath: "./testdata/test.db",
 				},
 				Common: &CommonCfg{
-					LogDir:   "./testdata/",
+					LogDir:   "./testdata",
 					LogMedia: "stdout",
 				},
 				DisableAPI: false,
@@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) {
 					ShareContext:          ptr.Of(false),
 					ConsoleManagement:     ptr.Of(false),
 				},
-				LogDir:   LogDirFullPath,
+				LogDir:   "./testdata",
 				LogMedia: "stdout",
 				OnlineClient: &OnlineApiClientCfg{
 					CredentialsFilePath: "./testdata/online-api-secrets.yaml",

pkg/csconfig/common.go

@@ -14,7 +14,7 @@ type CommonCfg struct {
 	LogMedia       string     `yaml:"log_media"`
 	LogDir         string     `yaml:"log_dir,omitempty"` //if LogMedia = file
 	LogLevel       *log.Level `yaml:"log_level"`
-	WorkingDir     string     `yaml:"working_dir,omitempty"` ///var/run
+	WorkingDir     string     `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
 	CompressLogs   *bool      `yaml:"compress_logs,omitempty"`
 	LogMaxSize     int        `yaml:"log_max_size,omitempty"`
 	LogMaxAge      int        `yaml:"log_max_age,omitempty"`
@@ -22,15 +22,18 @@ type CommonCfg struct {
 	ForceColorLogs bool       `yaml:"force_color_logs,omitempty"`
 }
 
-func (c *Config) LoadCommon() error {
+func (c *Config) loadCommon() error {
 	var err error
 	if c.Common == nil {
-		return fmt.Errorf("no common block provided in configuration file")
+		c.Common = &CommonCfg{}
+	}
+
+	if c.Common.LogMedia == "" {
+		c.Common.LogMedia = "stdout"
 	}
 
 	var CommonCleanup = []*string{
 		&c.Common.LogDir,
-		&c.Common.WorkingDir,
 	}
 	for _, k := range CommonCleanup {
 		if *k == "" {

pkg/csconfig/common_test.go

@@ -1,83 +0,0 @@
-package csconfig
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-)
-
-func TestLoadCommon(t *testing.T) {
-	pidDirPath := "./testdata"
-	LogDirFullPath, err := filepath.Abs("./testdata/log/")
-	require.NoError(t, err)
-
-	WorkingDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	tests := []struct {
-		name        string
-		input       *Config
-		expected    *CommonCfg
-		expectedErr string
-	}{
-		{
-			name: "basic valid configuration",
-			input: &Config{
-				Common: &CommonCfg{
-					Daemonize:  true,
-					PidDir:     "./testdata",
-					LogMedia:   "file",
-					LogDir:     "./testdata/log/",
-					WorkingDir: "./testdata/",
-				},
-			},
-			expected: &CommonCfg{
-				Daemonize:  true,
-				PidDir:     pidDirPath,
-				LogMedia:   "file",
-				LogDir:     LogDirFullPath,
-				WorkingDir: WorkingDirFullPath,
-			},
-		},
-		{
-			name: "empty working dir",
-			input: &Config{
-				Common: &CommonCfg{
-					Daemonize: true,
-					PidDir:    "./testdata",
-					LogMedia:  "file",
-					LogDir:    "./testdata/log/",
-				},
-			},
-			expected: &CommonCfg{
-				Daemonize: true,
-				PidDir:    pidDirPath,
-				LogMedia:  "file",
-				LogDir:    LogDirFullPath,
-			},
-		},
-		{
-			name:        "no common",
-			input:       &Config{},
-			expected:    nil,
-			expectedErr: "no common block provided in configuration file",
-		},
-	}
-
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadCommon()
-			cstest.RequireErrorContains(t, err, tc.expectedErr)
-			if tc.expectedErr != "" {
-				return
-			}
-
-			assert.Equal(t, tc.expected, tc.input.Common)
-		})
-	}
-}

pkg/csconfig/config.go

@@ -36,7 +36,7 @@ type Config struct {
 	PluginConfig *PluginCfg          `yaml:"plugin_config,omitempty"`
 	DisableAPI   bool                `yaml:"-"`
 	DisableAgent bool                `yaml:"-"`
-	Hub          *Hub                `yaml:"-"`
+	Hub          *HubCfg             `yaml:"-"`
 }
 
 func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
@@ -58,18 +58,49 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
 		// this is actually the "merged" yaml
 		return nil, "", fmt.Errorf("%s: %w", configFile, err)
 	}
+
+	if cfg.Prometheus == nil {
+		cfg.Prometheus = &PrometheusCfg{}
+	}
+
+	if cfg.Prometheus.ListenAddr == "" {
+		cfg.Prometheus.ListenAddr = "127.0.0.1"
+		log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr)
+	}
+
+	if cfg.Prometheus.ListenPort == 0 {
+		cfg.Prometheus.ListenPort = 6060
+		log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
+	}
+
+	if err = cfg.loadCommon(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadConfigurationPaths(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadHub(); err != nil {
+		return nil, "", err
+	}
+
+	if err = cfg.loadCSCLI(); err != nil {
+		return nil, "", err
+	}
+
 	return &cfg, configData, nil
 }
 
+// XXX: We must not have a different behavior with an empty vs a missing configuration file.
+// XXX: For this reason, all defaults have to come from NewConfig(). The following function should
+// XXX: be replaced
 func NewDefaultConfig() *Config {
 	logLevel := log.InfoLevel
 	commonCfg := CommonCfg{
 		Daemonize: false,
-		PidDir:    "/tmp/",
 		LogMedia:  "stdout",
-		//LogDir unneeded
 		LogLevel:   &logLevel,
-		WorkingDir: ".",
 	}
 	prometheus := PrometheusCfg{
 		Enabled: true,

pkg/csconfig/config_paths.go

@@ -15,21 +15,25 @@ type ConfigurationPaths struct {
 	NotificationDir    string `yaml:"notification_dir,omitempty"`
 }
 
-func (c *Config) LoadConfigurationPaths() error {
+func (c *Config) loadConfigurationPaths() error {
 	var err error
 	if c.ConfigPaths == nil {
+		// XXX: test me
 		return fmt.Errorf("no configuration paths provided")
 	}
 
 	if c.ConfigPaths.DataDir == "" {
+		// XXX: test me
 		return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
 	}
 
 	if c.ConfigPaths.HubDir == "" {
+		// XXX: test me
 		c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
 	}
 
 	if c.ConfigPaths.HubIndexFile == "" {
+		// XXX: test me
 		c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
 	}
 

pkg/csconfig/crowdsec_service.go

@@ -149,10 +149,6 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("loading api client: %s", err)
 	}
 
-	if err := c.LoadHub(); err != nil {
-		return fmt.Errorf("while loading hub: %w", err)
-	}
-
 	c.Crowdsec.ContextToSend = make(map[string][]string, 0)
 	fallback := false
 	if c.Crowdsec.ConsoleContextPath == "" {

pkg/csconfig/crowdsec_service_test.go

@@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) {
 	acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
 	require.NoError(t, err)
 
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
 	require.NoError(t, err)
 
@@ -66,10 +54,11 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        "",
 				ConsoleContextPath:        contextFileFullPath,
 				AcquisitionFilePath:       acquisFullPath,
-				ConfigDir:                 configDirFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
+				ConfigDir:                 "./testdata",
+				DataDir:                   "./data",
+				HubDir:                    "./hub",
+				// XXX: need to ensure a default here
+				HubIndexFile:              "",
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -109,10 +98,11 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        acquisDirFullPath,
 				AcquisitionFilePath:       acquisFullPath,
 				ConsoleContextPath:        contextFileFullPath,
-				ConfigDir:                 configDirFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
+				ConfigDir:                 "./testdata",
+				// XXX: need to ensure a default here
+				HubIndexFile:              "",
+				DataDir:                   "./data",
+				HubDir:                    "./hub",
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -141,7 +131,7 @@ func TestLoadCrowdsec(t *testing.T) {
 					},
 				},
 				Crowdsec: &CrowdsecServiceCfg{
-					ConsoleContextPath:        contextFileFullPath,
+					ConsoleContextPath:        "./testdata/context.yaml",
 					ConsoleContextValueLength: 10,
 				},
 			},
@@ -149,10 +139,11 @@ func TestLoadCrowdsec(t *testing.T) {
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				AcquisitionFilePath:       "",
-				ConfigDir:                 configDirFullPath,
-				HubIndexFile:              hubIndexFileFullPath,
-				DataDir:                   dataFullPath,
-				HubDir:                    hubFullPath,
+				ConfigDir:                 "./testdata",
+				// XXX: need to ensure a default here
+				HubIndexFile:              "",
+				DataDir:                   "./data",
+				HubDir:                    "./hub",
 				ConsoleContextPath:        contextFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,

pkg/csconfig/cscli.go

@@ -1,5 +1,9 @@
 package csconfig
 
+import (
+	"fmt"
+)
+
 /*cscli specific config, such as hub directory*/
 type CscliCfg struct {
 	Output             string            `yaml:"output,omitempty"`
@@ -15,17 +19,18 @@ type CscliCfg struct {
 	PrometheusUrl      string            `yaml:"prometheus_uri"`
 }
 
-func (c *Config) LoadCSCLI() error {
+func (c *Config) loadCSCLI() error {
 	if c.Cscli == nil {
 		c.Cscli = &CscliCfg{}
 	}
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
 	c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
 	c.Cscli.DataDir = c.ConfigPaths.DataDir
 	c.Cscli.HubDir = c.ConfigPaths.HubDir
 	c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
 
+	if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
+		c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
+	}
+
 	return nil
 }

pkg/csconfig/cscli_test.go

@@ -1,28 +1,14 @@
 package csconfig
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadCSCLI(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
@@ -38,26 +24,27 @@ func TestLoadCSCLI(t *testing.T) {
 					HubDir:       "./hub",
 					HubIndexFile: "./hub/.index.json",
 				},
+				Prometheus: &PrometheusCfg{
+					Enabled:    true,
+					Level:      "full",
+					ListenAddr: "127.0.0.1",
+					ListenPort: 6060,
+				},
 			},
 			expected: &CscliCfg{
-				ConfigDir:    configDirFullPath,
-				DataDir:      dataFullPath,
-				HubDir:       hubFullPath,
-				HubIndexFile: hubIndexFileFullPath,
+				ConfigDir:     "./testdata",
+				DataDir:       "./data",
+				HubDir:        "./hub",
+				HubIndexFile:  "./hub/.index.json",
+				PrometheusUrl: "http://127.0.0.1:6060/metrics",
 			},
 		},
-		{
-			name:        "no configuration path",
-			input:       &Config{},
-			expected:    &CscliCfg{},
-			expectedErr: "no configuration paths provided",
-		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadCSCLI()
+			err := tc.input.loadCSCLI()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return

pkg/csconfig/hub.go

@@ -1,19 +1,15 @@
 package csconfig
 
-/*cscli specific config, such as hub directory*/
-type Hub struct {
-	HubIndexFile   string
-	HubDir         string
-	InstallDir     string
-	InstallDataDir string
+// HubConfig holds the configuration for a hub
+type HubCfg struct {
+	HubIndexFile   string	// Path to the local index file
+	HubDir         string	// Where the hub items are downloaded
+	InstallDir     string	// Where to install items
+	InstallDataDir string	// Where to install data
 }
 
-func (c *Config) LoadHub() error {
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
-
-	c.Hub = &Hub{
+func (c *Config) loadHub() error {
+	c.Hub = &HubCfg{
 		HubIndexFile:   c.ConfigPaths.HubIndexFile,
 		HubDir:         c.ConfigPaths.HubDir,
 		InstallDir:     c.ConfigPaths.ConfigDir,

pkg/csconfig/hub_test.go

@@ -1,32 +1,18 @@
 package csconfig
 
 import (
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 
 	"github.com/crowdsecurity/go-cs-lib/cstest"
 )
 
 func TestLoadHub(t *testing.T) {
-	hubFullPath, err := filepath.Abs("./hub")
-	require.NoError(t, err)
-
-	dataFullPath, err := filepath.Abs("./data")
-	require.NoError(t, err)
-
-	configDirFullPath, err := filepath.Abs("./testdata")
-	require.NoError(t, err)
-
-	hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
-		expected    *Hub
+		expected    *HubCfg
 		expectedErr string
 	}{
 		{
@@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) {
 					HubIndexFile: "./hub/.index.json",
 				},
 			},
-			expected: &Hub{
-				HubDir:         hubFullPath,
-				HubIndexFile:   hubIndexFileFullPath,
-				InstallDir:     configDirFullPath,
-				InstallDataDir: dataFullPath,
+			expected: &HubCfg{
+				HubDir:         "./hub",
+				HubIndexFile:   "./hub/.index.json",
+				InstallDir:     "./testdata",
+				InstallDataDir: "./data",
 			},
 		},
-		{
-			name: "no data dir",
-			input: &Config{
-				ConfigPaths: &ConfigurationPaths{
-					ConfigDir:    "./testdata",
-					HubDir:       "./hub",
-					HubIndexFile: "./hub/.index.json",
-				},
-			},
-			expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
-		},
-		{
-			name:        "no configuration path",
-			input:       &Config{},
-			expectedErr: "no configuration paths provided",
-		},
 	}
 
 	for _, tc := range tests {
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadHub()
+			err := tc.input.loadHub()
 			cstest.RequireErrorContains(t, err, tc.expectedErr)
 			if tc.expectedErr != "" {
 				return

pkg/csconfig/prometheus.go

@@ -1,19 +1,8 @@
 package csconfig
 
-import "fmt"
-
 type PrometheusCfg struct {
 	Enabled    bool   `yaml:"enabled"`
 	Level      string `yaml:"level"` //aggregated|full
 	ListenAddr string `yaml:"listen_addr"`
 	ListenPort int    `yaml:"listen_port"`
 }
-
-func (c *Config) LoadPrometheus() error {
-	if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil {
-		if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
-			c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
-		}
-	}
-	return nil
-}

pkg/csconfig/prometheus_test.go

@@ -1,42 +0,0 @@
-package csconfig
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-)
-
-func TestLoadPrometheus(t *testing.T) {
-	tests := []struct {
-		name        string
-		input       *Config
-		expectedURL string
-		expectedErr string
-	}{
-		{
-			name: "basic valid configuration",
-			input: &Config{
-				Prometheus: &PrometheusCfg{
-					Enabled:    true,
-					Level:      "full",
-					ListenAddr: "127.0.0.1",
-					ListenPort: 6060,
-				},
-				Cscli: &CscliCfg{},
-			},
-			expectedURL: "http://127.0.0.1:6060",
-		},
-	}
-
-	for _, tc := range tests {
-		tc := tc
-		t.Run(tc.name, func(t *testing.T) {
-			err := tc.input.LoadPrometheus()
-			cstest.RequireErrorContains(t, err, tc.expectedErr)
-
-			require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl)
-		})
-	}
-}

pkg/csconfig/simulation.go

@@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
 }
 
 func (c *Config) LoadSimulation() error {
-
-	if err := c.LoadConfigurationPaths(); err != nil {
-		return err
-	}
-
 	simCfg := SimulationConfig{}
 	if c.ConfigPaths.SimulationFilePath == "" {
 		c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")

pkg/csconfig/simulation_test.go

@@ -2,7 +2,6 @@ package csconfig
 
 import (
 	"fmt"
-	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -12,12 +11,6 @@ import (
 )
 
 func TestSimulationLoading(t *testing.T) {
-	testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml")
-	require.NoError(t, err)
-
-	badYamlFullPath, err := filepath.Abs("./testdata/config.yaml")
-	require.NoError(t, err)
-
 	tests := []struct {
 		name        string
 		input       *Config
@@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage),
+			expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage),
 		},
 		{
 			name: "basic bad file content",
@@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+			expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
 		},
 		{
 			name: "basic bad file content",
@@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) {
 				},
 				Crowdsec: &CrowdsecServiceCfg{},
 			},
-			expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
+			expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
 		},
 	}
 

pkg/csconfig/testdata/config.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: false
   log_media: stdout
   log_level: info
-  working_dir: .
 prometheus:
   enabled: true
   level: full

pkg/cwhub/cwhub.go

@@ -1,66 +1,45 @@
+// Package cwhub is responsible for installing and upgrading the local hub files.
+//
+// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
+// and managing the dependencies and taints.
 package cwhub
 
 import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 
 	"github.com/enescakir/emoji"
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
 	"golang.org/x/mod/semver"
 )
 
-const (
-	HubIndexFile = ".index.json"
-
-	// managed item types
-	PARSERS       = "parsers"
-	PARSERS_OVFLW = "postoverflows"
-	SCENARIOS     = "scenarios"
-	COLLECTIONS   = "collections"
-	WAAP_RULES    = "waap-rules"
-)
-
 var (
-	ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS, WAAP_RULES}
-
 	ErrMissingReference = errors.New("Reference(s) missing in collection")
 
-	// XXX: can we remove these globals?
-	skippedLocal       = 0
-	skippedTainted     = 0
 	RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
 	HubBranch          = "master"
-	hubIdx             map[string]map[string]Item
 )
 
+// ItemVersion is used to detect the version of a given item
+// by comparing the hash of each version to the local file.
+// If the item does not match any known version, it is considered tainted.
 type ItemVersion struct {
-	Digest     string `json:"digest,omitempty"` // meow
-	Deprecated bool   `json:"deprecated,omitempty"`
+	Digest     string `json:"digest,omitempty"`     // meow
+	Deprecated bool   `json:"deprecated,omitempty"` // XXX: do we keep this?
 }
 
-type ItemHubStatus struct {
-	Name         string `json:"name"`
-	LocalVersion string `json:"local_version"`
-	LocalPath    string `json:"local_path"`
-	Description  string `json:"description"`
-	UTF8Status   string `json:"utf8_status"`
-	Status       string `json:"status"`
-}
-
-// Item can be: parser, scenario, collection..
+// Item represents an object managed in the hub. It can be a parser, scenario, collection..
 type Item struct {
 	// descriptive info
 	Type                 string   `json:"type,omitempty"                   yaml:"type,omitempty"`                   // parser|postoverflows|scenario|collection(|enrich)
 	Stage                string   `json:"stage,omitempty"                  yaml:"stage,omitempty"`                  // Stage for parser|postoverflow: s00-raw/s01-...
-	Name                 string   `json:"name,omitempty"`                                                           // as seen in .config.json, usually "author/name"
+	Name                 string   `json:"name,omitempty"`                                                           // as seen in .index.json, usually "author/name"
 	FileName             string   `json:"file_name,omitempty"`                                                      // the filename, ie. apache2-logs.yaml
-	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .config.json
-	Author               string   `json:"author,omitempty"`                                                         // as seen in .config.json
-	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`             // as seen in .config.json
+	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .index.json
+	Author               string   `json:"author,omitempty"`                                                         // as seen in .index.json
+	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`             // as seen in .index.json
 	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
 
 	// remote (hub) info
@@ -78,7 +57,7 @@ type Item struct {
 	Tainted      bool   `json:"tainted,omitempty"` // has it been locally modified
 	Local        bool   `json:"local,omitempty"`   // if it's a non versioned control one
 
-	// if it's a collection, it's not a single file
+	// if it's a collection, it can have sub items
 	Parsers       []string `json:"parsers,omitempty"       yaml:"parsers,omitempty"`
 	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
 	Scenarios     []string `json:"scenarios,omitempty"     yaml:"scenarios,omitempty"`
@@ -86,7 +65,9 @@ type Item struct {
 	WafRules      []string `json:"waap-rules,omitempty"    yaml:"waap-rules,omitempty"`
 }
 
-func (i *Item) status() (string, emoji.Emoji) {
+// Status returns the status of the item as a string and an emoji
+// ie. "enabled,update-available" and emoji.Warning
+func (i *Item) Status() (string, emoji.Emoji) {
 	status := "disabled"
 	ok := false
 
@@ -126,26 +107,14 @@ func (i *Item) status() (string, emoji.Emoji) {
 	return status, emo
 }
 
-func (i *Item) hubStatus() ItemHubStatus {
-	status, emo := i.status()
-
-	return ItemHubStatus{
-		Name:         i.Name,
-		LocalVersion: i.LocalVersion,
-		LocalPath:    i.LocalPath,
-		Description:  i.Description,
-		Status:       status,
-		UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
-	}
-}
-
 // versionStatus: semver requires 'v' prefix
 func (i *Item) versionStatus() int {
 	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
 }
 
+// GetItemMap returns the map of items for a given type
 func GetItemMap(itemType string) map[string]Item {
-	m, ok := hubIdx[itemType]
+	m, ok := hubIdx.Items[itemType]
 	if !ok {
 		return nil
 	}
@@ -153,7 +122,7 @@ func GetItemMap(itemType string) map[string]Item {
 	return m
 }
 
-// Given a FileInfo, extract the map key. Follow a symlink if necessary
+// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
 func itemKey(itemPath string) (string, error) {
 	f, err := os.Lstat(itemPath)
 	if err != nil {
@@ -200,6 +169,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) {
 	return &v, nil
 }
 
+// GetItem returns the item from hub based on its type and full name (author/name)
 func GetItem(itemType string, itemName string) *Item {
 	if m, ok := GetItemMap(itemType)[itemName]; ok {
 		return &m
@@ -208,10 +178,28 @@ func GetItem(itemType string, itemName string) *Item {
 	return nil
 }
 
+// GetItemNames returns the list of item (full) names for a given type
+// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
+// The names can be used to retrieve the item with GetItem()
+func GetItemNames(itemType string) []string {
+	m := GetItemMap(itemType)
+	if m == nil {
+		return nil
+	}
+
+	names := make([]string, 0, len(m))
+	for k := range m {
+		names = append(names, k)
+	}
+
+	return names
+}
+
+// AddItem adds an item to the hub index
 func AddItem(itemType string, item Item) error {
 	for _, itype := range ItemTypes {
 		if itype == itemType {
-			hubIdx[itemType][item.Name] = item
+			hubIdx.Items[itemType][item.Name] = item
 			return nil
 		}
 	}
@@ -219,17 +207,9 @@ func AddItem(itemType string, item Item) error {
 	return fmt.Errorf("ItemType %s is unknown", itemType)
 }
 
-func DisplaySummary() {
-	log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers,%d waf rules", len(hubIdx[COLLECTIONS]),
-		len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[WAAP_RULES]))
-
-	if skippedLocal > 0 || skippedTainted > 0 {
-		log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
-	}
-}
-
+// GetInstalledItems returns the list of installed items
 func GetInstalledItems(itemType string) ([]Item, error) {
-	items, ok := hubIdx[itemType]
+	items, ok := hubIdx.Items[itemType]
 	if !ok {
 		return nil, fmt.Errorf("no %s in hubIdx", itemType)
 	}
@@ -245,6 +225,7 @@ func GetInstalledItems(itemType string) ([]Item, error) {
 	return retItems, nil
 }
 
+// GetInstalledItemsAsString returns the names of the installed items
 func GetInstalledItemsAsString(itemType string) ([]string, error) {
 	items, err := GetInstalledItems(itemType)
 	if err != nil {
@@ -259,32 +240,3 @@ func GetInstalledItemsAsString(itemType string) ([]string, error) {
 
 	return retStr, nil
 }
-
-// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
-func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
-	if _, ok := hubIdx[itemType]; !ok {
-		log.Errorf("type %s doesn't exist", itemType)
-
-		return nil
-	}
-
-	ret := make([]ItemHubStatus, 0)
-
-	// remember, you do it for the user :)
-	for _, item := range hubIdx[itemType] {
-		if name != "" && name != item.Name {
-			// user has requested a specific name
-			continue
-		}
-		// Only enabled items ?
-		if !all && !item.Installed {
-			continue
-		}
-		// Check the item status
-		ret = append(ret, item.hubStatus())
-	}
-
-	sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
-
-	return ret
-}

pkg/cwhub/cwhub_test.go

@@ -53,7 +53,7 @@ func TestItemStatus(t *testing.T) {
 		item.Local = false
 		item.Tainted = false
 
-		txt, _ := item.status()
+		txt, _ := item.Status()
 		require.Equal(t, "enabled,update-available", txt)
 
 		item.Installed = false
@@ -61,7 +61,7 @@ func TestItemStatus(t *testing.T) {
 		item.Local = true
 		item.Tainted = false
 
-		txt, _ = item.status()
+		txt, _ = item.Status()
 		require.Equal(t, "disabled,local", txt)
 	}
 
@@ -121,7 +121,7 @@ func TestIndexDownload(t *testing.T) {
 }
 
 func getTestCfg() *csconfig.Config {
-	cfg := &csconfig.Config{Hub: &csconfig.Hub{}}
+	cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}}
 	cfg.Hub.InstallDir, _ = filepath.Abs("./install")
 	cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
 	cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
@@ -172,7 +172,7 @@ func envTearDown(cfg *csconfig.Config) {
 	}
 }
 
-func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
+func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
 	// Install the parser
 	err := DownloadLatest(cfg, &item, false, false)
 	require.NoError(t, err, "failed to download %s", item.Name)
@@ -180,9 +180,9 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
 
 	err = EnableItem(cfg, &item)
 	require.NoError(t, err, "failed to enable %s", item.Name)
@@ -190,11 +190,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 }
 
-func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
 
 	f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
 	require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
@@ -208,11 +208,11 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
 }
 
-func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
+func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
+	assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
 
 	// Update it + check status
 	err := DownloadLatest(cfg, &item, true, true)
@@ -222,12 +222,12 @@ func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
 }
 
-func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 
 	// Remove
 	err := DisableItem(cfg, &item, false, false)
@@ -238,9 +238,9 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, warns, "unexpected warnings : %+v", warns)
 
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
 
 	// Purge
 	err = DisableItem(cfg, &item, true, false)
@@ -251,8 +251,8 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, warns, "unexpected warnings : %+v", warns)
 
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
 }
 
 func TestInstallParser(t *testing.T) {
@@ -270,17 +270,15 @@ func TestInstallParser(t *testing.T) {
 
 	getHubIdxOrFail(t)
 	// map iteration is random by itself
-	for _, it := range hubIdx[PARSERS] {
+	for _, it := range hubIdx.Items[PARSERS] {
 		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
-		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
+		it = hubIdx.Items[PARSERS][it.Name]
 		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
-		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
+		it = hubIdx.Items[PARSERS][it.Name]
 		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
+		it = hubIdx.Items[PARSERS][it.Name]
 		testDisableItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
+		it = hubIdx.Items[PARSERS][it.Name]
 
 		break
 	}
@@ -301,19 +299,14 @@ func TestInstallCollection(t *testing.T) {
 
 	getHubIdxOrFail(t)
 	// map iteration is random by itself
-	for _, it := range hubIdx[COLLECTIONS] {
+	for _, it := range hubIdx.Items[COLLECTIONS] {
 		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
+		it = hubIdx.Items[COLLECTIONS][it.Name]
 		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
+		it = hubIdx.Items[COLLECTIONS][it.Name]
 		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
+		it = hubIdx.Items[COLLECTIONS][it.Name]
 		testDisableItem(cfg.Hub, t, it)
-
-		it = hubIdx[COLLECTIONS][it.Name]
-		x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
-		log.Infof("%+v", x)
-
 		break
 	}
 }

pkg/cwhub/download.go

@@ -19,20 +19,21 @@ import (
 
 var ErrIndexNotFound = fmt.Errorf("index not found")
 
-func UpdateHubIdx(hub *csconfig.Hub) error {
+// UpdateHubIdx downloads the latest version of the index and updates the one in memory
+func UpdateHubIdx(hub *csconfig.HubCfg) error {
 	bidx, err := DownloadHubIdx(hub)
 	if err != nil {
 		return fmt.Errorf("failed to download index: %w", err)
 	}
 
-	ret, err := LoadPkgIndex(bidx)
+	ret, err := ParseIndex(bidx)
 	if err != nil {
 		if !errors.Is(err, ErrMissingReference) {
 			return fmt.Errorf("failed to read index: %w", err)
 		}
 	}
 
-	hubIdx = ret
+	hubIdx = HubIndex{Items: ret}
 
 	if _, err := LocalSync(hub); err != nil {
 		return fmt.Errorf("failed to sync: %w", err)
@@ -41,7 +42,8 @@ func UpdateHubIdx(hub *csconfig.Hub) error {
 	return nil
 }
 
-func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
+// DownloadHubIdx downloads the latest version of the index and returns the content
+func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
 	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
 
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
@@ -96,7 +98,7 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
 }
 
 // DownloadLatest will download the latest version of Item to the tdir directory
-func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error {
+func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error {
 	var err error
 
 	log.Debugf("Downloading %s %s", target.Type, target.Name)
@@ -115,7 +117,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly
 	for idx, ptr := range tmp {
 		ptrtype := ItemTypes[idx]
 		for _, p := range ptr {
-			val, ok := hubIdx[ptrtype][p]
+			val, ok := hubIdx.Items[ptrtype][p]
 			if !ok {
 				return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
 			}
@@ -151,7 +153,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly
 				}
 			}
 
-			hubIdx[ptrtype][p] = val
+			hubIdx.Items[ptrtype][p] = val
 		}
 	}
 
@@ -163,7 +165,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly
 	return nil
 }
 
-func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
+func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
 	tdir := hub.HubDir
 
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
@@ -265,12 +267,13 @@ func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
 		return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
 	}
 
-	hubIdx[target.Type][target.Name] = *target
+	hubIdx.Items[target.Type][target.Name] = *target
 
 	return nil
 }
 
-func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
+// DownloadDataIfNeeded downloads the data files for an item
+func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error {
 	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
 
 	itemFile, err := os.Open(itemFilePath)
@@ -287,6 +290,7 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
 	return nil
 }
 
+// downloadData downloads the data files for an item
 func downloadData(dataFolder string, force bool, reader io.Reader) error {
 	var err error
 

pkg/cwhub/download_test.go

@@ -17,7 +17,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = "x"
 
-	ret, err := DownloadHubIdx(&csconfig.Hub{})
+	ret, err := DownloadHubIdx(&csconfig.HubCfg{})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
 		log.Errorf("unexpected error %s", err)
 	}
@@ -29,7 +29,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = "https://baddomain/%s/%s"
 
-	ret, err = DownloadHubIdx(&csconfig.Hub{})
+	ret, err = DownloadHubIdx(&csconfig.HubCfg{})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
 		log.Errorf("unexpected error %s", err)
 	}
@@ -41,7 +41,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = back
 
-	ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
+	ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"})
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
 		log.Errorf("unexpected error %s", err)
 	}

pkg/cwhub/helpers.go

@@ -12,12 +12,12 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 )
 
-// pick a hub branch corresponding to the current crowdsec version.
+// chooseHubBranch returns the branch name to use for the hub
+// It can be "master" or branch corresponding to the current crowdsec version
 func chooseHubBranch() string {
 	latest, err := cwversion.Latest()
 	if err != nil {
 		log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
-		//lint:ignore nilerr
 		return "master"
 	}
 
@@ -61,8 +61,9 @@ func SetHubBranch() {
 	log.Debugf("Using branch '%s' for the hub", HubBranch)
 }
 
-func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
-	item := GetItem(obtype, name)
+// InstallItem installs an item from the hub
+func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error {
+	item := GetItem(itemType, name)
 	if item == nil {
 		return fmt.Errorf("unable to retrieve item: %s", name)
 	}
@@ -80,7 +81,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
 		return fmt.Errorf("while downloading %s: %w", item.Name, err)
 	}
 
-	if err = AddItem(obtype, *item); err != nil {
+	if err = AddItem(itemType, *item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
@@ -94,7 +95,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
 		return fmt.Errorf("while enabling %s: %w", item.Name, err)
 	}
 
-	if err := AddItem(obtype, *item); err != nil {
+	if err := AddItem(itemType, *item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
@@ -103,29 +104,29 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
 	return nil
 }
 
-// XXX this must return errors instead of log.Fatal
-func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) {
+// RemoveItem removes one - or all - the items from the hub
+func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error {
 	if name != "" {
 		item := GetItem(itemType, name)
 		if item == nil {
-			log.Fatalf("unable to retrieve: %s", name)
+			return fmt.Errorf("can't find '%s' in %s", name, itemType)
 		}
 
 		err := DisableItem(csConfig.Hub, item, purge, forceAction)
 
 		if err != nil {
-			log.Fatalf("unable to disable %s : %v", item.Name, err)
+			return fmt.Errorf("unable to disable %s: %w", item.Name, err)
 		}
 
 		if err = AddItem(itemType, *item); err != nil {
-			log.Fatalf("unable to add %s: %v", item.Name, err)
+			return fmt.Errorf("unable to add %s: %w", item.Name, err)
 		}
 
-		return
+		return nil
 	}
 
 	if !all {
-		log.Fatal("removing item: no item specified")
+		return fmt.Errorf("removing item: no item specified")
 	}
 
 	disabled := 0
@@ -138,19 +139,22 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
 
 		err := DisableItem(csConfig.Hub, &v, purge, forceAction)
 		if err != nil {
-			log.Fatalf("unable to disable %s : %v", v.Name, err)
+			return fmt.Errorf("unable to disable %s: %w", v.Name, err)
 		}
 
 		if err := AddItem(itemType, v); err != nil {
-			log.Fatalf("unable to add %s: %v", v.Name, err)
+			return fmt.Errorf("unable to add %s: %w", v.Name, err)
 		}
 		disabled++
 	}
 
 	log.Infof("Disabled %d items", disabled)
+
+	return nil
 }
 
-func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) {
+// UpgradeConfig upgrades an item from the hub
+func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error {
 	updated := 0
 	found := false
 
@@ -165,17 +169,17 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 		}
 
 		if !v.Downloaded {
-			log.Warningf("%s : not downloaded, please install.", v.Name)
+			log.Warningf("%s: not downloaded, please install.", v.Name)
 			continue
 		}
 
 		found = true
 
 		if v.UpToDate {
-			log.Infof("%s : up-to-date", v.Name)
+			log.Infof("%s: up-to-date", v.Name)
 
 			if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil {
-				log.Fatalf("%s : download failed : %v", v.Name, err)
+				return fmt.Errorf("%s: download failed: %w", v.Name, err)
 			}
 
 			if !force {
@@ -184,7 +188,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 		}
 
 		if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil {
-			log.Fatalf("%s : download failed : %v", v.Name, err)
+			return fmt.Errorf("%s: download failed: %w", v.Name, err)
 		}
 
 		if !v.UpToDate {
@@ -202,14 +206,14 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 		}
 
 		if err := AddItem(itemType, v); err != nil {
-			log.Fatalf("unable to add %s: %v", v.Name, err)
+			return fmt.Errorf("unable to add %s: %w", v.Name, err)
 		}
 	}
 
 	if !found && name == "" {
 		log.Infof("No %s installed, nothing to upgrade", itemType)
 	} else if !found {
-		log.Errorf("Item '%s' not found in hub", name)
+		log.Errorf("can't find '%s' in %s", name, itemType)
 	} else if updated == 0 && found {
 		if name == "" {
 			log.Infof("All %s are already up-to-date", itemType)
@@ -219,4 +223,6 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 	} else if updated != 0 {
 		log.Infof("Upgraded %d items", updated)
 	}
+
+	return nil
 }

pkg/cwhub/helpers_test.go

@@ -15,19 +15,19 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
 	// fresh install of collection
 	getHubIdxOrFail(t)
 
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
 
 	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
 
 	// This is the scenario that gets added in next version of collection
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
@@ -40,16 +40,17 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
 
 	getHubIdxOrFail(t)
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
 
-	UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	require.NoError(t, err)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 }
 
 // Install a collection, disable a scenario.
@@ -61,36 +62,39 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	// fresh install of collection
 	getHubIdxOrFail(t)
 
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 
 	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
-	RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
+	err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
+	require.NoError(t, err)
+
 	getHubIdxOrFail(t)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
 
-	if err := UpdateHubIdx(cfg.Hub); err != nil {
+	if err = UpdateHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to download index : %s", err)
 	}
 
-	UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	require.NoError(t, err)
 
 	getHubIdxOrFail(t)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 }
 
 func getHubIdxOrFail(t *testing.T) {
@@ -109,51 +113,55 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
 	// fresh install of collection
 	getHubIdxOrFail(t)
 
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 
 	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
-	RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
+	err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
+	require.NoError(t, err)
+
 	getHubIdxOrFail(t)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
 	// we just removed. Nor should it install the newly added scenario
 	pushUpdateToCollectionInHub()
 
-	if err := UpdateHubIdx(cfg.Hub); err != nil {
+	if err = UpdateHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to download index : %s", err)
 	}
 
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	getHubIdxOrFail(t)
 
-	UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	require.NoError(t, err)
+
 	getHubIdxOrFail(t)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 }
 
 func assertCollectionDepsInstalled(t *testing.T, collection string) {
 	t.Helper()
 
-	c := hubIdx[COLLECTIONS][collection]
+	c := hubIdx.Items[COLLECTIONS][collection]
 	require.NoError(t, CollecDepsCheck(&c))
 }
 

pkg/cwhub/hubindex.go

@@ -0,0 +1,105 @@
+package cwhub
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	HubIndexFile = ".index.json"
+
+	// managed item types
+	COLLECTIONS   = "collections"
+	PARSERS       = "parsers"
+	POSTOVERFLOWS = "postoverflows"
+	SCENARIOS     = "scenarios"
+	WAAP_RULES    = "waap-rules"
+)
+
+var (
+	// XXX: The order is important, as it is used to construct the
+	//      index tree in memory --> collections must be last
+	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS, WAAP_RULES}
+	hubIdx    = HubIndex{}
+)
+
+type HubItems map[string]map[string]Item
+
+// HubIndex represents the runtime status of the hub (parsed items, etc.)
+// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared
+type HubIndex struct {
+	Items          HubItems
+	skippedLocal   int
+	skippedTainted int
+}
+
+// displaySummary prints a total count of the hub items
+func (h HubIndex) displaySummary() {
+	msg := "Loaded: "
+	for itemType := range h.Items {
+		msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
+	}
+	log.Info(strings.Trim(msg, ", "))
+
+	if h.skippedLocal > 0 || h.skippedTainted > 0 {
+		log.Infof("unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted)
+	}
+}
+
+// DisplaySummary prints a total count of the hub items.
+// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton
+func DisplaySummary() {
+	hubIdx.displaySummary()
+}
+
+// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections
+func ParseIndex(buff []byte) (HubItems, error) {
+	var (
+		RawIndex     HubItems
+		missingItems []string
+	)
+
+	if err := json.Unmarshal(buff, &RawIndex); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal index: %w", err)
+	}
+
+	log.Debugf("%d item types in hub index", len(ItemTypes))
+
+	// Iterate over the different types to complete the struct
+	for _, itemType := range ItemTypes {
+		log.Tracef("%s: %d items", itemType, len(RawIndex[itemType]))
+
+		for name, item := range RawIndex[itemType] {
+			item.Name = name
+			item.Type = itemType
+			x := strings.Split(item.RemotePath, "/")
+			item.FileName = x[len(x)-1]
+			RawIndex[itemType][name] = item
+
+			if itemType != COLLECTIONS {
+				continue
+			}
+
+			// if it's a collection, check its sub-items are present
+			// XXX should be done later
+			for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
+				ptrtype := ItemTypes[idx]
+				for _, p := range ptr {
+					if _, ok := RawIndex[ptrtype][p]; !ok {
+						log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
+						missingItems = append(missingItems, p)
+					}
+				}
+			}
+		}
+	}
+
+	if len(missingItems) > 0 {
+		return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
+	}
+
+	return RawIndex, nil
+}

pkg/cwhub/install.go

@@ -10,7 +10,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
-func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
+func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) {
 	itempath := hub.HubDir + "/" + target.RemotePath
 
 	// disable hub file
@@ -20,13 +20,13 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
 
 	target.Downloaded = false
 	log.Infof("Removed source file [%s]: %s", target.Name, itempath)
-	hubIdx[target.Type][target.Name] = target
+	hubIdx.Items[target.Type][target.Name] = target
 
 	return target, nil
 }
 
 // DisableItem to disable an item managed by the hub, removes the symlink if purge is true
-func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error {
+func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error {
 	var err error
 
 	// already disabled, noop unless purge
@@ -54,7 +54,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error
 		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
-				if val, ok := hubIdx[ptrtype][p]; ok {
+				if val, ok := hubIdx.Items[ptrtype][p]; ok {
 					// check if the item doesn't belong to another collection before removing it
 					toRemove := true
 
@@ -130,14 +130,14 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error
 		}
 	}
 
-	hubIdx[target.Type][target.Name] = *target
+	hubIdx.Items[target.Type][target.Name] = *target
 
 	return nil
 }
 
 // creates symlink between actual config file at hub.HubDir and hub.ConfigDir
 // Handles collections recursively
-func EnableItem(hub *csconfig.Hub, target *Item) error {
+func EnableItem(hub *csconfig.HubCfg, target *Item) error {
 	var err error
 
 	parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
@@ -172,7 +172,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error {
 		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
-				val, ok := hubIdx[ptrtype][p]
+				val, ok := hubIdx.Items[ptrtype][p]
 				if !ok {
 					return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
 				}
@@ -208,7 +208,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error {
 
 	log.Infof("Enabled %s : %s", target.Type, target.Name)
 	target.Installed = true
-	hubIdx[target.Type][target.Name] = *target
+	hubIdx.Items[target.Type][target.Name] = *target
 
 	return nil
 }

pkg/cwhub/loader.go

@@ -2,7 +2,6 @@ package cwhub
 
 import (
 	"crypto/sha256"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -67,7 +66,7 @@ type Walker struct {
 	installdir string
 }
 
-func NewWalker(hub *csconfig.Hub) Walker {
+func NewWalker(hub *csconfig.HubCfg) Walker {
 	return Walker{
 		hubdir:     hub.HubDir,
 		installdir: hub.InstallDir,
@@ -98,7 +97,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 		//.../hub/scenarios/crowdsec/ssh_bf.yaml
 		//.../hub/profiles/crowdsec/linux.yaml
 		if len(subs) < 4 {
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
+			return itemFileInfo{}, false, fmt.Errorf("path is too short : %s (%d)", path, len(subs))
 		}
 
 		ret.fname = subs[len(subs)-1]
@@ -108,7 +107,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 	} else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec/<type>/...
 		log.Tracef("in install dir")
 		if len(subs) < 3 {
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
+			return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
 		}
 		///.../config/parser/stage/file.yaml
 		///.../config/postoverflow/stage/file.yaml
@@ -134,7 +133,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 	} else if ret.stage == WAAP_RULES {
 		ret.ftype = WAAP_RULES
 		ret.stage = ""
-	} else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW {
+	} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
 		// its a PARSER / PARSER_OVFLW with a stage
 		return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
 	}
@@ -144,7 +143,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 	return ret, inhub, nil
 }
 
-func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
+func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 	var (
 		local   bool
 		hubpath string
@@ -201,12 +200,12 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 	// if it's not a symlink and not in hub, it's a local file, don't bother
 	if local && !inhub {
 		log.Tracef("%s is a local file, skip", path)
-		skippedLocal++
+		hubIdx.skippedLocal++
 		//	log.Infof("local scenario, skip.")
 
 		_, fileName := filepath.Split(path)
 
-		hubIdx[info.ftype][info.fname] = Item{
+		hubIdx.Items[info.ftype][info.fname] = Item{
 			Name:      info.fname,
 			Stage:     info.stage,
 			Installed: true,
@@ -225,7 +224,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 
 	match := false
 
-	for name, item := range hubIdx[info.ftype] {
+	for name, item := range hubIdx.Items[info.ftype] {
 		log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
 
 		if info.fname != item.FileName {
@@ -307,7 +306,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 		if !match {
 			log.Tracef("got tainted match for %s: %s", item.Name, path)
 
-			skippedTainted++
+			hubIdx.skippedTainted++
 			// the file and the stage is right, but the hash is wrong, it has been tainted by user
 			if !inhub {
 				item.LocalPath = path
@@ -320,14 +319,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 			item.LocalHash = sha
 		}
 
-		// update the entry if appropriate
-		// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
-		// 	fmt.Printf("Updating %s", k)
-		// 	hubIdx[ftype][k] = v
-		// } else if !inhub {
-
-		// } else if
-		hubIdx[info.ftype][name] = item
+		hubIdx.Items[info.ftype][name] = item
 
 		return nil
 	}
@@ -353,9 +345,9 @@ func CollecDepsCheck(v *Item) error {
 	for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
 		sliceType := ItemTypes[idx]
 		for _, subName := range itemSlice {
-			subItem, ok := hubIdx[sliceType][subName]
+			subItem, ok := hubIdx.Items[sliceType][subName]
 			if !ok {
-				log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name)
+				return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name)
 			}
 
 			log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
@@ -375,7 +367,7 @@ func CollecDepsCheck(v *Item) error {
 					return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
 				}
 
-				hubIdx[sliceType][subName] = subItem
+				hubIdx.Items[sliceType][subName] = subItem
 			}
 
 			// propagate the state of sub-items to set
@@ -406,7 +398,7 @@ func CollecDepsCheck(v *Item) error {
 				subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
 			}
 
-			hubIdx[sliceType][subName] = subItem
+			hubIdx.Items[sliceType][subName] = subItem
 
 			log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
 		}
@@ -415,23 +407,23 @@ func CollecDepsCheck(v *Item) error {
 	return nil
 }
 
-func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
+func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
 	warnings := []string{}
 
-	// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
+	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
 	for _, scan := range ItemTypes {
 		cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
 		if err != nil {
 			log.Errorf("failed %s : %s", cpath, err)
 		}
 
-		err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit)
+		err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit)
 		if err != nil {
 			return warnings, err
 		}
 	}
 
-	for name, item := range hubIdx[COLLECTIONS] {
+	for name, item := range hubIdx.Items[COLLECTIONS] {
 		if !item.Installed {
 			continue
 		}
@@ -441,7 +433,7 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
 		case 0: // latest
 			if err := CollecDepsCheck(&item); err != nil {
 				warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
-				hubIdx[COLLECTIONS][name] = item
+				hubIdx.Items[COLLECTIONS][name] = item
 			}
 		case 1: // not up-to-date
 			warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
@@ -456,9 +448,9 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
 }
 
 // Updates the info from HubInit() with the local state
-func LocalSync(hub *csconfig.Hub) ([]string, error) {
-	skippedLocal = 0
-	skippedTainted = 0
+func LocalSync(hub *csconfig.HubCfg) ([]string, error) {
+	hubIdx.skippedLocal = 0
+	hubIdx.skippedTainted = 0
 
 	warnings, err := SyncDir(hub, hub.InstallDir)
 	if err != nil {
@@ -473,7 +465,7 @@ func LocalSync(hub *csconfig.Hub) ([]string, error) {
 	return warnings, nil
 }
 
-func GetHubIdx(hub *csconfig.Hub) error {
+func GetHubIdx(hub *csconfig.HubCfg) error {
 	if hub == nil {
 		return fmt.Errorf("no configuration found for hub")
 	}
@@ -485,7 +477,7 @@ func GetHubIdx(hub *csconfig.Hub) error {
 		return fmt.Errorf("unable to read index file: %w", err)
 	}
 
-	ret, err := LoadPkgIndex(bidx)
+	ret, err := ParseIndex(bidx)
 	if err != nil {
 		if !errors.Is(err, ErrMissingReference) {
 			return fmt.Errorf("unable to load existing index: %w", err)
@@ -495,7 +487,7 @@ func GetHubIdx(hub *csconfig.Hub) error {
 		return err
 	}
 
-	hubIdx = ret
+	hubIdx = HubIndex{Items: ret}
 
 	_, err = LocalSync(hub)
 	if err != nil {
@@ -504,52 +496,3 @@ func GetHubIdx(hub *csconfig.Hub) error {
 
 	return nil
 }
-
-// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections
-func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
-	var (
-		RawIndex     map[string]map[string]Item
-		missingItems []string
-	)
-
-	if err := json.Unmarshal(buff, &RawIndex); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal index: %w", err)
-	}
-
-	log.Debugf("%d item types in hub index", len(ItemTypes))
-
-	// Iterate over the different types to complete the struct
-	for _, itemType := range ItemTypes {
-		log.Tracef("%d item", len(RawIndex[itemType]))
-
-		for name, item := range RawIndex[itemType] {
-			item.Name = name
-			item.Type = itemType
-			x := strings.Split(item.RemotePath, "/")
-			item.FileName = x[len(x)-1]
-			RawIndex[itemType][name] = item
-
-			if itemType != COLLECTIONS {
-				continue
-			}
-
-			// if it's a collection, check its sub-items are present
-			// XXX should be done later
-			for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections, item.WafRules} {
-				ptrtype := ItemTypes[idx]
-				for _, p := range ptr {
-					if _, ok := RawIndex[ptrtype][p]; !ok {
-						log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
-						missingItems = append(missingItems, p)
-					}
-				}
-			}
-		}
-	}
-
-	if len(missingItems) > 0 {
-		return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
-	}
-
-	return RawIndex, nil
-}

pkg/hubtest/coverage.go

@@ -27,12 +27,12 @@ type ScenarioCoverage struct {
 
 func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
 	var coverage []ParserCoverage
-	if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok {
+	if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
 		return coverage, fmt.Errorf("no parsers in hub index")
 	}
 	//populate from hub, iterate in alphabetical order
 	var pkeys []string
-	for pname := range h.HubIndex.Data[cwhub.PARSERS] {
+	for pname := range h.HubIndex.Items[cwhub.PARSERS] {
 		pkeys = append(pkeys, pname)
 	}
 	sort.Strings(pkeys)
@@ -100,12 +100,12 @@ func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
 
 func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) {
 	var coverage []ScenarioCoverage
-	if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok {
+	if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
 		return coverage, fmt.Errorf("no scenarios in hub index")
 	}
 	//populate from hub, iterate in alphabetical order
 	var pkeys []string
-	for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] {
+	for scenarioName := range h.HubIndex.Items[cwhub.SCENARIOS] {
 		pkeys = append(pkeys, scenarioName)
 	}
 	sort.Strings(pkeys)

pkg/hubtest/hubtest.go

@@ -18,7 +18,7 @@ type HubTest struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *HubIndex
+	HubIndex               *cwhub.HubIndex
 	Tests                  []*HubTestItem
 }
 
@@ -62,7 +62,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 	}
 
 	// load hub index
-	hubIndex, err := cwhub.LoadPkgIndex(bidx)
+	hubIndex, err := cwhub.ParseIndex(bidx)
 	if err != nil {
 		return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err)
 	}
@@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 		TemplateConfigPath:     templateConfigFilePath,
 		TemplateProfilePath:    templateProfilePath,
 		TemplateSimulationPath: templateSimulationPath,
-		HubIndex:               &HubIndex{Data: hubIndex},
+		HubIndex:               &cwhub.HubIndex{Items: hubIndex},
 	}, nil
 }
 

pkg/hubtest/hubtest_item.go

@@ -25,10 +25,6 @@ type HubTestItemConfig struct {
 	OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
 }
 
-type HubIndex struct {
-	Data map[string]map[string]cwhub.Item
-}
-
 type HubTestItem struct {
 	Name string
 	Path string
@@ -43,7 +39,7 @@ type HubTestItem struct {
 	RuntimeConfigFilePath     string
 	RuntimeProfileFilePath    string
 	RuntimeSimulationFilePath string
-	RuntimeHubConfig          *csconfig.Hub
+	RuntimeHubConfig          *csconfig.HubCfg
 
 	ResultsPath          string
 	ParserResultFile     string
@@ -56,7 +52,7 @@ type HubTestItem struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *HubIndex
+	HubIndex               *cwhub.HubIndex
 
 	Config *HubTestItemConfig
 
@@ -121,7 +117,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
 		ParserResultFile:          filepath.Join(resultPath, ParserResultFileName),
 		ScenarioResultFile:        filepath.Join(resultPath, ScenarioResultFileName),
 		BucketPourResultFile:      filepath.Join(resultPath, BucketPourResultFileName),
-		RuntimeHubConfig: &csconfig.Hub{
+		RuntimeHubConfig: &csconfig.HubCfg{
 			HubDir:         runtimeHubFolder,
 			HubIndexFile:   hubTest.HubIndexFile,
 			InstallDir:     runtimeFolder,
@@ -148,7 +144,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 		var parserDirDest string
-		if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok {
+		if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
 			parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
@@ -232,7 +228,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 		var scenarioDirDest string
-		if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok {
+		if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
 			scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
@@ -301,7 +297,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 		var postoverflowDirDest string
-		if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok {
+		if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
 			postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
@@ -423,7 +419,7 @@ func (t *HubTestItem) InstallHub() error {
 	}
 
 	// install data for postoverflows if needed
-	ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW)
+	ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS)
 	for postoverflowName, item := range ret {
 		if item.Installed {
 			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {

pkg/parser/unix_parser.go

@@ -64,7 +64,7 @@ func NewParsers() *Parsers {
 		StageFiles:      make([]Stagefile, 0),
 		PovfwStageFiles: make([]Stagefile, 0),
 	}
-	for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
+	for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
 		for _, hubParserItem := range cwhub.GetItemMap(itemType) {
 			if hubParserItem.Installed {
 				stagefile := Stagefile{
@@ -74,7 +74,7 @@ func NewParsers() *Parsers {
 				if itemType == cwhub.PARSERS {
 					parsers.StageFiles = append(parsers.StageFiles, stagefile)
 				}
-				if itemType == cwhub.PARSERS_OVFLW {
+				if itemType == cwhub.POSTOVERFLOWS {
 					parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
 				}
 			}

pkg/setup/install.go

@@ -52,10 +52,6 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 		return err
 	}
 
-	if err := csConfig.LoadHub(); err != nil {
-		return fmt.Errorf("loading hub: %w", err)
-	}
-
 	cwhub.SetHubBranch()
 
 	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
@@ -121,7 +117,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
+				if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
 					return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
 				}
 			}

test/bats/01_crowdsec.bats

@@ -55,16 +55,16 @@ teardown() {
     assert_stderr --partial "unable to create database client: unknown database type 'meh'"
 }
 
-@test "crowdsec - bad configuration (empty/missing common section)" {
+@test "crowdsec - default logging configuration (empty/missing common section)" {
     config_set '.common={}'
-    rune -1 "${CROWDSEC}"
+    rune -124 timeout 1s "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "unable to load configuration: common section is empty"
+    assert_stderr --partial "Starting processing data"
 
     config_set 'del(.common)'
-    rune -1 "${CROWDSEC}"
+    rune -124 timeout 1s "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "unable to load configuration: common section is empty"
+    assert_stderr --partial "Starting processing data"
 }
 
 @test "CS_LAPI_SECRET not strong enough" {

test/bats/08_metrics.bats

@@ -25,8 +25,7 @@ teardown() {
 @test "cscli metrics (crowdsec not running)" {
     rune -1 cscli metrics
     # crowdsec is down
-    assert_stderr --partial "failed to fetch prometheus metrics"
-    assert_stderr --partial "connect: connection refused"
+    assert_stderr --partial 'failed to fetch prometheus metrics: executing GET request for URL \"http://127.0.0.1:6060/metrics\" failed: Get \"http://127.0.0.1:6060/metrics\": dial tcp 127.0.0.1:6060: connect: connection refused'
 }
 
 @test "cscli metrics (bad configuration)" {
@@ -43,18 +42,20 @@ teardown() {
 
 @test "cscli metrics (missing listen_addr)" {
     config_set 'del(.prometheus.listen_addr)'
-    rune -1 cscli metrics
-    assert_stderr --partial "no prometheus url, please specify"
+    rune -0 ./instance-crowdsec start
+    rune -0 cscli metrics --debug
+    assert_stderr --partial "prometheus.listen_addr is empty, defaulting to 127.0.0.1"
 }
 
 @test "cscli metrics (missing listen_port)" {
-    config_set 'del(.prometheus.listen_addr)'
-    rune -1 cscli metrics
-    assert_stderr --partial "no prometheus url, please specify"
+    config_set 'del(.prometheus.listen_port)'
+    rune -0 ./instance-crowdsec start
+    rune -0 cscli metrics --debug
+    assert_stderr --partial "prometheus.listen_port is empty or zero, defaulting to 6060"
 }
 
 @test "cscli metrics (missing prometheus section)" {
     config_set 'del(.prometheus)'
     rune -1 cscli metrics
-    assert_stderr --partial "prometheus section missing, can't show metrics"
+    assert_stderr --partial "prometheus is not enabled, can't show metrics"
 }

test/bats/20_collections.bats

@@ -1,145 +0,0 @@
-#!/usr/bin/env bats
-# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
-
-set -u
-
-setup_file() {
-    load "../lib/setup_file.sh"
-}
-
-teardown_file() {
-    load "../lib/teardown_file.sh"
-}
-
-setup() {
-    load "../lib/setup.sh"
-    ./instance-data load
-    ./instance-crowdsec start
-}
-
-teardown() {
-    ./instance-crowdsec stop
-}
-
-#----------
-
-@test "we can list collections" {
-    rune -0 cscli collections list
-}
-
-@test "there are 2 collections (linux and sshd)" {
-    rune -0 cscli collections list -o json
-    rune -0 jq '.collections | length' <(output)
-    assert_output 2
-}
-
-@test "can install a collection (as a regular user) and remove it" {
-    # collection is not installed
-    rune -0 cscli collections list -o json
-    rune -0 jq -r '.collections[].name' <(output)
-    refute_line "crowdsecurity/mysql"
-
-    # we install it
-    rune -0 cscli collections install crowdsecurity/mysql -o human
-    assert_stderr --partial "Enabled crowdsecurity/mysql"
-
-    # it has been installed
-    rune -0 cscli collections list -o json
-    rune -0 jq -r '.collections[].name' <(output)
-    assert_line "crowdsecurity/mysql"
-
-    # we install it
-    rune -0 cscli collections remove crowdsecurity/mysql -o human
-    assert_stderr --partial "Removed symlink [crowdsecurity/mysql]"
-
-    # it has been removed
-    rune -0 cscli collections list -o json
-    rune -0 jq -r '.collections[].name' <(output)
-    refute_line "crowdsecurity/mysql"
-}
-
-@test "must use --force to remove a collection that belongs to another, which becomes tainted" {
-    # we expect no error since we may have multiple collections, some removed and some not
-    rune -0 cscli collections remove crowdsecurity/sshd
-    assert_stderr --partial "crowdsecurity/sshd belongs to other collections"
-    assert_stderr --partial "[crowdsecurity/linux]"
-
-    rune -0 cscli collections remove crowdsecurity/sshd --force
-    assert_stderr --partial "Removed symlink [crowdsecurity/sshd]"
-    rune -0 cscli collections inspect crowdsecurity/linux -o json
-    rune -0 jq -r '.tainted' <(output)
-    assert_output "true"
-}
-
-@test "can remove a collection" {
-    rune -0 cscli collections remove crowdsecurity/linux
-    assert_stderr --partial "Removed"
-    assert_stderr --regexp   ".*for the new configuration to be effective."
-    rune -0 cscli collections inspect crowdsecurity/linux -o human
-    assert_line 'installed: false'
-}
-
-@test "collections delete is an alias for collections remove" {
-    rune -0 cscli collections delete crowdsecurity/linux
-    assert_stderr --partial "Removed"
-    assert_stderr --regexp   ".*for the new configuration to be effective."
-}
-
-@test "removing a collection that does not exist is noop" {
-    rune -0 cscli collections remove crowdsecurity/apache2
-    refute_stderr --partial "Removed"
-    assert_stderr --regexp   ".*for the new configuration to be effective."
-}
-
-@test "can remove a removed collection" {
-    rune -0 cscli collections install crowdsecurity/mysql
-    rune -0 cscli collections remove crowdsecurity/mysql
-    assert_stderr --partial "Removed"
-    rune -0 cscli collections remove crowdsecurity/mysql
-    refute_stderr --partial "Removed"
-}
-
-@test "can remove all collections" {
-    # we may have this too, from package installs
-    rune cscli parsers delete crowdsecurity/whitelists
-    rune -0 cscli collections remove --all
-    assert_stderr --partial "Removed symlink [crowdsecurity/sshd]"
-    assert_stderr --partial "Removed symlink [crowdsecurity/linux]"
-    rune -0 cscli hub list -o json
-    assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}'
-    rune -0 cscli collections remove --all
-    assert_stderr --partial 'Disabled 0 items'
-}
-
-@test "a taint bubbles up to the top collection" {
-    coll=crowdsecurity/nginx
-    subcoll=crowdsecurity/base-http-scenarios
-    scenario=crowdsecurity/http-crawl-non_statics
-
-    # install a collection with dependencies
-    rune -0 cscli collections install "$coll"
-
-    # the collection, subcollection and scenario are installed and not tainted
-    # we have to default to false because tainted is (as of 1.4.6) returned
-    # only when true
-    rune -0 cscli collections inspect "$coll" -o json
-    rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
-    rune -0 cscli collections inspect "$subcoll" -o json
-    rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
-    rune -0 cscli scenarios inspect "$scenario" -o json
-    rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
-
-    # we taint the scenario
-    HUB_DIR=$(config_get '.config_paths.hub_dir')
-    yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml"
-
-    # the collection, subcollection and scenario are now tainted
-    rune -0 cscli scenarios inspect "$scenario" -o json
-    rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
-    rune -0 cscli collections inspect "$subcoll" -o json
-    rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
-    rune -0 cscli collections inspect "$coll" -o json
-    rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
-}
-
-# TODO test download-only

test/bats/20_hub_collections.bats

@@ -0,0 +1,319 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    HUB_DIR=$(config_get '.config_paths.hub_dir')
+    export HUB_DIR
+    CONFIG_DIR=$(config_get '.config_paths.config_dir')
+    export CONFIG_DIR
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    load "../lib/bats-file/load.bash"
+    ./instance-data load
+    hub_uninstall_all
+    hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
+    echo "$hub_min" >"$HUB_DIR/.index.json"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "cscli collections list" {
+    # no items
+    rune -0 cscli collections list
+    assert_output --partial "COLLECTIONS"
+    rune -0 cscli collections list -o json
+    assert_json '{collections:[]}'
+    rune -0 cscli collections list -o raw
+    assert_output 'name,status,version,description'
+
+    # some items
+    rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb
+
+    rune -0 cscli collections list
+    assert_output --partial crowdsecurity/sshd
+    assert_output --partial crowdsecurity/smb
+    rune -0 grep -c enabled <(output)
+    assert_output "2"
+
+    rune -0 cscli collections list -o json
+    assert_output --partial crowdsecurity/sshd
+    assert_output --partial crowdsecurity/smb
+    rune -0 jq '.collections | length' <(output)
+    assert_output "2"
+
+    rune -0 cscli collections list -o raw
+    assert_output --partial crowdsecurity/sshd
+    assert_output --partial crowdsecurity/smb
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+}
+
+@test "cscli collections list -a" {
+    expected=$(jq <"$HUB_DIR/.index.json" -r '.collections | length')
+
+    rune -0 cscli collections list -a
+    rune -0 grep -c disabled <(output)
+    assert_output "$expected"
+
+    rune -0 cscli collections list -o json -a
+    rune -0 jq '.collections | length' <(output)
+    assert_output "$expected"
+
+    rune -0 cscli collections list -o raw -a
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "$expected"
+}
+
+
+@test "cscli collections list [collection]..." {
+    rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb
+
+    # list one item
+    rune -0 cscli collections list crowdsecurity/sshd
+    assert_output --partial "crowdsecurity/sshd"
+    refute_output --partial "crowdsecurity/smb"
+
+    # list multiple items
+    rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb
+    assert_output --partial "crowdsecurity/sshd"
+    assert_output --partial "crowdsecurity/smb"
+
+    rune -0 cscli collections list crowdsecurity/sshd -o json
+    rune -0 jq '.collections | length' <(output)
+    assert_output "1"
+    rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o json
+    rune -0 jq '.collections | length' <(output)
+    assert_output "2"
+
+    rune -0 cscli collections list crowdsecurity/sshd -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "1"
+    rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+}
+
+@test "cscli collections list [collection]... (not installed / not existing)" {
+    skip "not implemented yet"
+    # not installed
+    rune -1 cscli collections list crowdsecurity/sshd
+    # not existing
+    rune -1 cscli collections list blahblah/blahblah
+}
+
+@test "cscli collections install [collection]..." {
+    rune -1 cscli collections install
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+
+    # not in hub
+    rune -1 cscli collections install crowdsecurity/blahblah
+    assert_stderr --partial "can't find 'crowdsecurity/blahblah' in collections"
+
+    # simple install
+    rune -0 cscli collections install crowdsecurity/sshd
+    rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics
+    assert_output --partial 'crowdsecurity/sshd'
+    assert_output --partial 'installed: true'
+
+    # autocorrect
+    rune -1 cscli collections install crowdsecurity/ssshd
+    assert_stderr --partial "can't find 'crowdsecurity/ssshd' in collections, did you mean crowdsecurity/sshd?"
+
+    # install multiple
+    rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb
+    rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics
+    assert_output --partial 'crowdsecurity/sshd'
+    assert_output --partial 'installed: true'
+    rune -0 cscli collections inspect crowdsecurity/smb --no-metrics
+    assert_output --partial 'crowdsecurity/smb'
+    assert_output --partial 'installed: true'
+}
+
+@test "cscli collections install [collection]... (file location and download-only)" {
+    # simple install
+    rune -0 cscli collections install crowdsecurity/linux --download-only
+    rune -0 cscli collections inspect crowdsecurity/linux --no-metrics
+    assert_output --partial 'crowdsecurity/linux'
+    assert_output --partial 'installed: false'
+    assert_file_exists "$HUB_DIR/collections/crowdsecurity/linux.yaml"
+    assert_file_not_exists "$CONFIG_DIR/collections/linux.yaml"
+
+    rune -0 cscli collections install crowdsecurity/linux
+    assert_file_exists "$CONFIG_DIR/collections/linux.yaml"
+}
+
+
+@test "cscli collections inspect [collection]..." {
+    rune -1 cscli collections inspect
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+    ./instance-crowdsec start
+
+    rune -1 cscli collections inspect blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in collections"
+
+    # one item
+    rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics
+    assert_line 'type: collections'
+    assert_line 'name: crowdsecurity/sshd'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: collections/crowdsecurity/sshd.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # one item, with metrics
+    rune -0 cscli collections inspect crowdsecurity/sshd
+    assert_line --partial 'Current metrics:'
+
+    # one item, json
+    rune -0 cscli collections inspect crowdsecurity/sshd -o json
+    rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output)
+    # XXX: .installed is missing -- not false
+    assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null]'
+
+    # one item, raw
+    rune -0 cscli collections inspect crowdsecurity/sshd -o raw
+    assert_line 'type: collections'
+    assert_line 'name: crowdsecurity/sshd'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: collections/crowdsecurity/sshd.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # multiple items
+    rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb --no-metrics
+    assert_output --partial 'crowdsecurity/sshd'
+    assert_output --partial 'crowdsecurity/smb'
+    rune -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+
+    # multiple items, with metrics
+    rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb
+    rune -0 grep -c 'Current metrics:' <(output)
+    assert_output "2"
+
+    # multiple items, json
+    rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o json
+    rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output)
+    assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",null]]'
+
+    # multiple items, raw
+    rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw
+    assert_output --partial 'crowdsecurity/sshd'
+    assert_output --partial 'crowdsecurity/smb'
+    run -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+}
+
+@test "cscli collections remove [collection]..." {
+    rune -1 cscli collections remove
+    assert_stderr --partial "specify at least one collection to remove or '--all'"
+
+    rune -1 cscli collections remove blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in collections"
+
+    # XXX: we can however remove a real item if it's not installed, or already removed
+    rune -0 cscli collections remove crowdsecurity/sshd
+
+    # install, then remove, check files
+    rune -0 cscli collections install crowdsecurity/sshd
+    assert_file_exists "$CONFIG_DIR/collections/sshd.yaml"
+    rune -0 cscli collections remove crowdsecurity/sshd
+    assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml"
+
+    # delete is an alias for remove
+    rune -0 cscli collections install crowdsecurity/sshd
+    assert_file_exists "$CONFIG_DIR/collections/sshd.yaml"
+    rune -0 cscli collections delete crowdsecurity/sshd
+    assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml"
+
+    # purge
+    assert_file_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml"
+    rune -0 cscli collections remove crowdsecurity/sshd --purge
+    assert_file_not_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml"
+
+    rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb
+
+    # --all
+    rune -0 cscli collections list -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+
+    rune -0 cscli collections remove --all
+
+    rune -0 cscli collections list -o raw
+    rune -1 grep -vc 'name,status,version,description' <(output)
+    assert_output "0"
+}
+
+@test "cscli collections upgrade [collection]..." {
+    rune -1 cscli collections upgrade
+    assert_stderr --partial "specify at least one collection to upgrade or '--all'"
+
+    # XXX: should this return 1 instead of log.Error?
+    rune -0 cscli collections upgrade blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in collections"
+
+    # XXX: same message if the item exists but is not installed, this is confusing
+    rune -0 cscli collections upgrade crowdsecurity/sshd
+    assert_stderr --partial "can't find 'crowdsecurity/sshd' in collections"
+
+    # hash of an empty file
+    sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+    # add version 0.0 to the hub
+    new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/sshd":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
+    echo "$new_hub" >"$HUB_DIR/.index.json"
+ 
+    rune -0 cscli collections install crowdsecurity/sshd
+
+    # bring the file to v0.0
+    truncate -s 0 "$CONFIG_DIR/collections/sshd.yaml"
+    rune -0 cscli collections inspect crowdsecurity/sshd -o json
+    rune -0 jq -e '.local_version=="0.0"' <(output)
+
+    # upgrade
+    rune -0 cscli collections upgrade crowdsecurity/sshd
+    rune -0 cscli collections inspect crowdsecurity/sshd -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # taint
+    echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
+    # XXX: should return error
+    rune -0 cscli collections upgrade crowdsecurity/sshd
+    assert_stderr --partial "crowdsecurity/sshd is tainted, --force to overwrite"
+    rune -0 cscli collections inspect crowdsecurity/sshd -o json
+    rune -0 jq -e '.local_version=="?"' <(output)
+
+    # force upgrade with taint
+    rune -0 cscli collections upgrade crowdsecurity/sshd --force
+    rune -0 cscli collections inspect crowdsecurity/sshd -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # multiple items
+    rune -0 cscli collections install crowdsecurity/smb
+    echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
+    echo "dirty" >"$CONFIG_DIR/collections/smb.yaml"
+    rune -0 cscli collections list -o json
+    rune -0 jq -e '[.collections[].local_version]==["?","?"]' <(output)
+    rune -0 cscli collections upgrade crowdsecurity/sshd crowdsecurity/smb
+    rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output)
+
+    # upgrade all
+    echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
+    echo "dirty" >"$CONFIG_DIR/collections/smb.yaml"
+    rune -0 cscli collections upgrade --all
+    rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output)
+}

test/bats/20_hub_collections_dep.bats

@@ -0,0 +1,86 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    HUB_DIR=$(config_get '.config_paths.hub_dir')
+    export HUB_DIR
+    CONFIG_DIR=$(config_get '.config_paths.config_dir')
+    export CONFIG_DIR
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    load "../lib/bats-file/load.bash"
+    ./instance-data load
+    hub_uninstall_all
+    hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
+    echo "$hub_min" >"$HUB_DIR/.index.json"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "cscli collections (dependencies)" {
+    # inject a dependency: smb requires sshd
+    hub_dep=$(jq <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}')
+    echo "$hub_dep" >"$HUB_DIR/.index.json"
+
+    # verify that installing smb brings sshd
+    rune -0 cscli collections install crowdsecurity/smb
+    rune -0 cscli collections list -o json
+    rune -0 jq -e '[.collections[].name]==["crowdsecurity/smb","crowdsecurity/sshd"]' <(output)
+
+    # verify that removing smb removes sshd too
+    rune -0 cscli collections remove crowdsecurity/smb
+    rune -0 cscli collections list -o json
+    rune -0 jq -e '.collections | length == 0' <(output)
+
+    # we can't remove sshd without --force
+    rune -0 cscli collections install crowdsecurity/smb
+    # XXX: should this be an error?
+    rune -0 cscli collections remove crowdsecurity/sshd
+    assert_stderr --partial "crowdsecurity/sshd belongs to other collections: [crowdsecurity/smb]"
+    assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this sub collection"
+    rune -0 cscli collections list -o json
+    rune -0 jq -c '[.collections[].name]' <(output)
+    assert_json '["crowdsecurity/smb","crowdsecurity/sshd"]'
+
+    # use the --force
+    rune -0 cscli collections remove crowdsecurity/sshd --force
+    rune -0 cscli collections list -o json
+    rune -0 jq -c '[.collections[].name]' <(output)
+    assert_json '["crowdsecurity/smb"]'
+
+    # and now smb is tainted!
+    rune -0 cscli collections inspect crowdsecurity/smb -o json
+    rune -0 jq -e '.tainted//false==true' <(output)
+    rune -0 cscli collections remove crowdsecurity/smb --force
+
+    # empty
+    rune -0 cscli collections list -o json
+    rune -0 jq -e '.collections | length == 0' <(output)
+
+    # reinstall
+    rune -0 cscli collections install crowdsecurity/smb --force
+
+    # taint on sshd means smb is tainted as well
+    rune -0 cscli collections inspect crowdsecurity/smb -o json
+    jq -e '.tainted//false==false' <(output)
+    echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
+    rune -0 cscli collections inspect crowdsecurity/smb -o json
+    jq -e '.tainted//false==true' <(output)
+
+    # now we can't remove smb without --force
+    rune -1 cscli collections remove crowdsecurity/smb
+    assert_stderr --partial "unable to disable crowdsecurity/smb: crowdsecurity/smb is tainted, use '--force' to overwrite"
+}

test/bats/20_hub_parsers.bats

@@ -0,0 +1,417 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    HUB_DIR=$(config_get '.config_paths.hub_dir')
+    export HUB_DIR
+    CONFIG_DIR=$(config_get '.config_paths.config_dir')
+    export CONFIG_DIR
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    load "../lib/bats-file/load.bash"
+    ./instance-data load
+    hub_uninstall_all
+    # XXX: remove all "content" fields from the index, to make sure
+    # XXX: we don't rely on it in any way
+    hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
+    echo "$hub_min" >"$HUB_DIR/.index.json"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "cscli parsers list" {
+    # no items
+    rune -0 cscli parsers list
+    assert_output --partial "PARSERS"
+    rune -0 cscli parsers list -o json
+    assert_json '{parsers:[]}'
+    rune -0 cscli parsers list -o raw
+    assert_output 'name,status,version,description'
+
+    # some items
+    rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth
+
+    rune -0 cscli parsers list
+    assert_output --partial crowdsecurity/whitelists
+    assert_output --partial crowdsecurity/windows-auth
+    rune -0 grep -c enabled <(output)
+    assert_output "2"
+
+    rune -0 cscli parsers list -o json
+    assert_output --partial crowdsecurity/whitelists
+    assert_output --partial crowdsecurity/windows-auth
+    rune -0 jq '.parsers | length' <(output)
+    assert_output "2"
+
+    rune -0 cscli parsers list -o raw
+    assert_output --partial crowdsecurity/whitelists
+    assert_output --partial crowdsecurity/windows-auth
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+}
+
+@test "cscli parsers list -a" {
+    expected=$(jq <"$HUB_DIR/.index.json" -r '.parsers | length')
+
+    rune -0 cscli parsers list -a
+    rune -0 grep -c disabled <(output)
+    assert_output "$expected"
+
+    rune -0 cscli parsers list -o json -a
+    rune -0 jq '.parsers | length' <(output)
+    assert_output "$expected"
+
+    rune -0 cscli parsers list -o raw -a
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "$expected"
+}
+
+@test "cscli parsers list [parser]..." {
+    # non-existent
+    rune -1 cscli parsers install foo/bar
+    assert_stderr --partial "can't find 'foo/bar' in parsers"
+
+    # not installed
+    rune -0 cscli parsers list crowdsecurity/whitelists
+    assert_output --regexp 'crowdsecurity/whitelists.*disabled'
+
+    # install two items
+    rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth
+
+    # list an installed item
+    rune -0 cscli parsers list crowdsecurity/whitelists
+    assert_output --regexp "crowdsecurity/whitelists.*enabled"
+    refute_output --partial "crowdsecurity/windows-auth"
+
+    # list multiple installed and non installed items
+    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs
+    assert_output --partial "crowdsecurity/whitelists"
+    assert_output --partial "crowdsecurity/windows-auth"
+    assert_output --partial "crowdsecurity/traefik-logs"
+
+    rune -0 cscli parsers list crowdsecurity/whitelists -o json
+    rune -0 jq '.parsers | length' <(output)
+    assert_output "1"
+    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json
+    rune -0 jq '.parsers | length' <(output)
+    assert_output "3"
+
+    rune -0 cscli parsers list crowdsecurity/whitelists -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "1"
+    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "3"
+}
+
+@test "cscli parsers install [parser]..." {
+    rune -1 cscli parsers install
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+
+    # not in hub
+    rune -1 cscli parsers install crowdsecurity/blahblah
+    assert_stderr --partial "can't find 'crowdsecurity/blahblah' in parsers"
+
+    # simple install
+    rune -0 cscli parsers install crowdsecurity/whitelists
+    rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics
+    assert_output --partial 'crowdsecurity/whitelists'
+    assert_output --partial 'installed: true'
+
+    # autocorrect
+    rune -1 cscli parsers install crowdsecurity/sshd-logz
+    assert_stderr --partial "can't find 'crowdsecurity/sshd-logz' in parsers, did you mean crowdsecurity/sshd-logs?"
+
+    # install multiple
+    rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs
+    rune -0 cscli parsers inspect crowdsecurity/pgsql-logs --no-metrics
+    assert_output --partial 'crowdsecurity/pgsql-logs'
+    assert_output --partial 'installed: true'
+    rune -0 cscli parsers inspect crowdsecurity/postfix-logs --no-metrics
+    assert_output --partial 'crowdsecurity/postfix-logs'
+    assert_output --partial 'installed: true'
+}
+
+@test "cscli parsers install [parser]... (file location and download-only)" {
+    # simple install
+    rune -0 cscli parsers install crowdsecurity/whitelists --download-only
+    rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics
+    assert_output --partial 'crowdsecurity/whitelists'
+    assert_output --partial 'installed: false'
+    assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml"
+    assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+
+    rune -0 cscli parsers install crowdsecurity/whitelists
+    assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+}
+
+# XXX: test install with --force
+# XXX: test install with --ignore
+
+@test "cscli parsers inspect [parser]..." {
+    rune -1 cscli parsers inspect
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+    ./instance-crowdsec start
+
+    rune -1 cscli parsers inspect blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in parsers"
+
+    # one item
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics
+    assert_line 'type: parsers'
+    assert_line 'stage: s01-parse'
+    assert_line 'name: crowdsecurity/sshd-logs'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # one item, with metrics
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs
+    assert_line --partial 'Current metrics:'
+
+    # one item, json
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json
+    rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output)
+    # XXX: .installed is missing -- not false
+    assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null]'
+
+    # one item, raw
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw
+    assert_line 'type: parsers'
+    assert_line 'stage: s01-parse'
+    assert_line 'name: crowdsecurity/sshd-logs'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # multiple items
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists --no-metrics
+    assert_output --partial 'crowdsecurity/sshd-logs'
+    assert_output --partial 'crowdsecurity/whitelists'
+    rune -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+
+    # multiple items, with metrics
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists
+    rune -0 grep -c 'Current metrics:' <(output)
+    assert_output "2"
+
+    # multiple items, json
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json
+    rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output)
+    assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",null]]'
+
+    # multiple items, raw
+    rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw
+    assert_output --partial 'crowdsecurity/sshd-logs'
+    assert_output --partial 'crowdsecurity/whitelists'
+    run -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+}
+
+@test "cscli parsers remove [parser]..." {
+    rune -1 cscli parsers remove
+    assert_stderr --partial "specify at least one parser to remove or '--all'"
+
+    rune -1 cscli parsers remove blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in parsers"
+
+    # XXX: we can however remove a real item if it's not installed, or already removed
+    rune -0 cscli parsers remove crowdsecurity/whitelists
+
+    # XXX: have the --force ignore uninstalled items
+    # XXX: maybe also with --purge
+
+    # install, then remove, check files
+    rune -0 cscli parsers install crowdsecurity/whitelists
+    assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+    rune -0 cscli parsers remove crowdsecurity/whitelists
+    assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+
+    # delete is an alias for remove
+    rune -0 cscli parsers install crowdsecurity/whitelists
+    assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+    rune -0 cscli parsers delete crowdsecurity/whitelists
+    assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+
+    # purge
+    assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml"
+    rune -0 cscli parsers remove crowdsecurity/whitelists --purge
+    assert_file_not_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml"
+
+    rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth
+
+    # --all
+    rune -0 cscli parsers list -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+
+    rune -0 cscli parsers remove --all
+
+    rune -0 cscli parsers list -o raw
+    rune -1 grep -vc 'name,status,version,description' <(output)
+    assert_output "0"
+}
+
+@test "cscli parsers upgrade [parser]..." {
+    rune -1 cscli parsers upgrade
+    assert_stderr --partial "specify at least one parser to upgrade or '--all'"
+
+    # XXX: should this return 1 instead of log.Error?
+    rune -0 cscli parsers upgrade blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in parsers"
+
+    # XXX: same message if the item exists but is not installed, this is confusing
+    rune -0 cscli parsers upgrade crowdsecurity/whitelists
+    assert_stderr --partial "can't find 'crowdsecurity/whitelists' in parsers"
+
+    # hash of an empty file
+    sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+    # add version 0.0 to the hub
+    new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {parsers:{"crowdsecurity/whitelists":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
+    echo "$new_hub" >"$HUB_DIR/.index.json"
+ 
+    rune -0 cscli parsers install crowdsecurity/whitelists
+
+    # bring the file to v0.0
+    truncate -s 0 "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+    rune -0 cscli parsers inspect crowdsecurity/whitelists -o json
+    rune -0 jq -e '.local_version=="0.0"' <(output)
+
+    # upgrade
+    rune -0 cscli parsers upgrade crowdsecurity/whitelists
+    rune -0 cscli parsers inspect crowdsecurity/whitelists -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # taint
+    echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+    # XXX: should return error
+    rune -0 cscli parsers upgrade crowdsecurity/whitelists
+    assert_stderr --partial "crowdsecurity/whitelists is tainted, --force to overwrite"
+    rune -0 cscli parsers inspect crowdsecurity/whitelists -o json
+    rune -0 jq -e '.local_version=="?"' <(output)
+
+    # force upgrade with taint
+    rune -0 cscli parsers upgrade crowdsecurity/whitelists --force
+    rune -0 cscli parsers inspect crowdsecurity/whitelists -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # multiple items
+    rune -0 cscli parsers install crowdsecurity/windows-auth
+    echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+    echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml"
+    rune -0 cscli parsers list -o json
+    rune -0 jq -e '[.parsers[].local_version]==["?","?"]' <(output)
+    rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/windows-auth
+    rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output)
+
+    # upgrade all
+    echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
+    echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml"
+    rune -0 cscli parsers upgrade --all
+    rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output)
+}
+
+
+
+#@test "must use --force to remove a collection that belongs to another, which becomes tainted" {
+#    # we expect no error since we may have multiple collections, some removed and some not
+#    rune -0 cscli collections remove crowdsecurity/sshd
+#    assert_stderr --partial "crowdsecurity/sshd belongs to other collections"
+#    assert_stderr --partial "[crowdsecurity/linux]"
+#
+#    rune -0 cscli collections remove crowdsecurity/sshd --force
+#    assert_stderr --partial "Removed symlink [crowdsecurity/sshd]"
+#    rune -0 cscli collections inspect crowdsecurity/linux -o json
+#    rune -0 jq -r '.tainted' <(output)
+#    assert_output "true"
+#}
+#
+#@test "can remove a collection" {
+#    rune -0 cscli collections remove crowdsecurity/linux
+#    assert_stderr --partial "Removed"
+#    assert_stderr --regexp   ".*for the new configuration to be effective."
+#    rune -0 cscli collections inspect crowdsecurity/linux -o human --no-metrics
+#    assert_line 'installed: false'
+#}
+#
+#@test "collections delete is an alias for collections remove" {
+#    rune -0 cscli collections delete crowdsecurity/linux
+#    assert_stderr --partial "Removed"
+#    assert_stderr --regexp   ".*for the new configuration to be effective."
+#}
+#
+#@test "removing a collection that does not exist is noop" {
+#    rune -0 cscli collections remove crowdsecurity/apache2
+#    refute_stderr --partial "Removed"
+#    assert_stderr --regexp   ".*for the new configuration to be effective."
+#}
+#
+#@test "can remove a removed collection" {
+#    rune -0 cscli collections install crowdsecurity/mysql
+#    rune -0 cscli collections remove crowdsecurity/mysql
+#    assert_stderr --partial "Removed"
+#    rune -0 cscli collections remove crowdsecurity/mysql
+#    refute_stderr --partial "Removed"
+#}
+#
+#@test "can remove all collections" {
+#    # we may have this too, from package installs
+#    rune cscli parsers delete crowdsecurity/whitelists
+#    rune -0 cscli collections remove --all
+#    assert_stderr --partial "Removed symlink [crowdsecurity/sshd]"
+#    assert_stderr --partial "Removed symlink [crowdsecurity/linux]"
+#    rune -0 cscli hub list -o json
+#    assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}'
+#    rune -0 cscli collections remove --all
+#    assert_stderr --partial 'Disabled 0 items'
+#}
+#
+#@test "a taint bubbles up to the top collection" {
+#    coll=crowdsecurity/nginx
+#    subcoll=crowdsecurity/base-http-scenarios
+#    scenario=crowdsecurity/http-crawl-non_statics
+#
+#    # install a collection with dependencies
+#    rune -0 cscli collections install "$coll"
+#
+#    # the collection, subcollection and scenario are installed and not tainted
+#    # we have to default to false because tainted is (as of 1.4.6) returned
+#    # only when true
+#    rune -0 cscli collections inspect "$coll" -o json
+#    rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
+#    rune -0 cscli collections inspect "$subcoll" -o json
+#    rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
+#    rune -0 cscli scenarios inspect "$scenario" -o json
+#    rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
+#
+#    # we taint the scenario
+#    HUB_DIR=$(config_get '.config_paths.hub_dir')
+#    yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml"
+#
+#    # the collection, subcollection and scenario are now tainted
+#    rune -0 cscli scenarios inspect "$scenario" -o json
+#    rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
+#    rune -0 cscli collections inspect "$subcoll" -o json
+#    rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
+#    rune -0 cscli collections inspect "$coll" -o json
+#    rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
+#}
+#
+## TODO test download-only

test/bats/20_hub_postoverflows.bats

@@ -0,0 +1,321 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    HUB_DIR=$(config_get '.config_paths.hub_dir')
+    export HUB_DIR
+    CONFIG_DIR=$(config_get '.config_paths.config_dir')
+    export CONFIG_DIR
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    load "../lib/bats-file/load.bash"
+    ./instance-data load
+    hub_uninstall_all
+    hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
+    echo "$hub_min" >"$HUB_DIR/.index.json"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "cscli postoverflows list" {
+    # no items
+    rune -0 cscli postoverflows list
+    assert_output --partial "POSTOVERFLOWS"
+    rune -0 cscli postoverflows list -o json
+    assert_json '{postoverflows:[]}'
+    rune -0 cscli postoverflows list -o raw
+    assert_output 'name,status,version,description'
+
+    # some items
+    rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist
+
+    rune -0 cscli postoverflows list
+    assert_output --partial crowdsecurity/rdns
+    assert_output --partial crowdsecurity/cdn-whitelist
+    rune -0 grep -c enabled <(output)
+    assert_output "2"
+
+    rune -0 cscli postoverflows list -o json
+    assert_output --partial crowdsecurity/rdns
+    assert_output --partial crowdsecurity/cdn-whitelist
+    rune -0 jq '.postoverflows | length' <(output)
+    assert_output "2"
+
+    rune -0 cscli postoverflows list -o raw
+    assert_output --partial crowdsecurity/rdns
+    assert_output --partial crowdsecurity/cdn-whitelist
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+}
+
+@test "cscli postoverflows list -a" {
+    expected=$(jq <"$HUB_DIR/.index.json" -r '.postoverflows | length')
+
+    rune -0 cscli postoverflows list -a
+    rune -0 grep -c disabled <(output)
+    assert_output "$expected"
+
+    rune -0 cscli postoverflows list -o json -a
+    rune -0 jq '.postoverflows | length' <(output)
+    assert_output "$expected"
+
+    rune -0 cscli postoverflows list -o raw -a
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "$expected"
+}
+
+
+@test "cscli postoverflows list [scenario]..." {
+    rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist
+
+    # list one item
+    rune -0 cscli postoverflows list crowdsecurity/rdns
+    assert_output --partial "crowdsecurity/rdns"
+    refute_output --partial "crowdsecurity/cdn-whitelist"
+
+    # list multiple items
+    rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist
+    assert_output --partial "crowdsecurity/rdns"
+    assert_output --partial "crowdsecurity/cdn-whitelist"
+
+    rune -0 cscli postoverflows list crowdsecurity/rdns -o json
+    rune -0 jq '.postoverflows | length' <(output)
+    assert_output "1"
+    rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json
+    rune -0 jq '.postoverflows | length' <(output)
+    assert_output "2"
+
+    rune -0 cscli postoverflows list crowdsecurity/rdns -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "1"
+    rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+}
+
+@test "cscli postoverflows list [scenario]... (not installed / not existing)" {
+    skip "not implemented yet"
+    # not installed
+    rune -1 cscli postoverflows list crowdsecurity/rdns
+    # not existing
+    rune -1 cscli postoverflows list blahblah/blahblah
+}
+
+@test "cscli postoverflows install [scenario]..." {
+    rune -1 cscli postoverflows install
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+
+    # not in hub
+    rune -1 cscli postoverflows install crowdsecurity/blahblah
+    assert_stderr --partial "can't find 'crowdsecurity/blahblah' in postoverflows"
+
+    # simple install
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics
+    assert_output --partial 'crowdsecurity/rdns'
+    assert_output --partial 'installed: true'
+
+    # autocorrect
+    rune -1 cscli postoverflows install crowdsecurity/rdnf
+    assert_stderr --partial "can't find 'crowdsecurity/rdnf' in postoverflows, did you mean crowdsecurity/rdns?"
+
+    # install multiple
+    rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics
+    assert_output --partial 'crowdsecurity/rdns'
+    assert_output --partial 'installed: true'
+    rune -0 cscli postoverflows inspect crowdsecurity/cdn-whitelist --no-metrics
+    assert_output --partial 'crowdsecurity/cdn-whitelist'
+    assert_output --partial 'installed: true'
+}
+
+@test "cscli postoverflows install [postoverflow]... (file location and download-only)" {
+    # simple install
+    rune -0 cscli postoverflows install crowdsecurity/rdns --download-only
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics
+    assert_output --partial 'crowdsecurity/rdns'
+    assert_output --partial 'installed: false'
+    assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml"
+    assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+    assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+}
+
+
+@test "cscli postoverflows inspect [scenario]..." {
+    rune -1 cscli postoverflows inspect
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+    ./instance-crowdsec start
+
+    rune -1 cscli postoverflows inspect blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows"
+
+    # one item
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics
+    assert_line 'type: postoverflows'
+    assert_line 'stage: s00-enrich'
+    assert_line 'name: crowdsecurity/rdns'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # one item, with metrics
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns
+    assert_line --partial 'Current metrics:'
+
+    # one item, json
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
+    rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output)
+    # XXX: .installed is missing -- not false
+    assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null]'
+
+    # one item, raw
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw
+    assert_line 'type: postoverflows'
+    assert_line 'stage: s00-enrich'
+    assert_line 'name: crowdsecurity/rdns'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # multiple items
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist --no-metrics
+    assert_output --partial 'crowdsecurity/rdns'
+    assert_output --partial 'crowdsecurity/cdn-whitelist'
+    rune -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+
+    # multiple items, with metrics
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist
+    rune -0 grep -c 'Current metrics:' <(output)
+    assert_output "2"
+
+    # multiple items, json
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json
+    rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output)
+    assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",null]]'
+
+    # multiple items, raw
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw
+    assert_output --partial 'crowdsecurity/rdns'
+    assert_output --partial 'crowdsecurity/cdn-whitelist'
+    run -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+}
+
+@test "cscli postoverflows remove [postoverflow]..." {
+    rune -1 cscli postoverflows remove
+    assert_stderr --partial "specify at least one postoverflow to remove or '--all'"
+
+    rune -1 cscli postoverflows remove blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows"
+
+    # XXX: we can however remove a real item if it's not installed, or already removed
+    rune -0 cscli postoverflows remove crowdsecurity/rdns
+
+    # install, then remove, check files
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+    assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+    rune -0 cscli postoverflows remove crowdsecurity/rdns
+    assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+
+    # delete is an alias for remove
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+    assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+    rune -0 cscli postoverflows delete crowdsecurity/rdns
+    assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+
+    # purge
+    assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml"
+    rune -0 cscli postoverflows remove crowdsecurity/rdns --purge
+    assert_file_not_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml"
+
+    rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist
+
+    # --all
+    rune -0 cscli postoverflows list -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+
+    rune -0 cscli postoverflows remove --all
+
+    rune -0 cscli postoverflows list -o raw
+    rune -1 grep -vc 'name,status,version,description' <(output)
+    assert_output "0"
+}
+
+@test "cscli postoverflows upgrade [postoverflow]..." {
+    rune -1 cscli postoverflows upgrade
+    assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'"
+
+    # XXX: should this return 1 instead of log.Error?
+    rune -0 cscli postoverflows upgrade blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows"
+
+    # XXX: same message if the item exists but is not installed, this is confusing
+    rune -0 cscli postoverflows upgrade crowdsecurity/rdns
+    assert_stderr --partial "can't find 'crowdsecurity/rdns' in postoverflows"
+
+    # hash of an empty file
+    sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+    # add version 0.0 to the hub
+    new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {postoverflows:{"crowdsecurity/rdns":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
+    echo "$new_hub" >"$HUB_DIR/.index.json"
+ 
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+
+    # bring the file to v0.0
+    truncate -s 0 "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
+    rune -0 jq -e '.local_version=="0.0"' <(output)
+
+    # upgrade
+    rune -0 cscli postoverflows upgrade crowdsecurity/rdns
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # taint
+    echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+    # XXX: should return error
+    rune -0 cscli postoverflows upgrade crowdsecurity/rdns
+    assert_stderr --partial "crowdsecurity/rdns is tainted, --force to overwrite"
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
+    rune -0 jq -e '.local_version=="?"' <(output)
+
+    # force upgrade with taint
+    rune -0 cscli postoverflows upgrade crowdsecurity/rdns --force
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # multiple items
+    rune -0 cscli postoverflows install crowdsecurity/cdn-whitelist
+    echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+    echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml"
+    rune -0 cscli postoverflows list -o json
+    rune -0 jq -e '[.postoverflows[].local_version]==["?","?"]' <(output)
+    rune -0 cscli postoverflows upgrade crowdsecurity/rdns crowdsecurity/cdn-whitelist
+    rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output)
+
+    # upgrade all
+    echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
+    echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml"
+    rune -0 cscli postoverflows upgrade --all
+    rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output)
+}

test/bats/20_hub_scenarios.bats

@@ -0,0 +1,321 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+    HUB_DIR=$(config_get '.config_paths.hub_dir')
+    export HUB_DIR
+    CONFIG_DIR=$(config_get '.config_paths.config_dir')
+    export CONFIG_DIR
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+    load "../lib/bats-file/load.bash"
+    ./instance-data load
+    hub_uninstall_all
+    hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
+    echo "$hub_min" >"$HUB_DIR/.index.json"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "cscli scenarios list" {
+    # no items
+    rune -0 cscli scenarios list
+    assert_output --partial "SCENARIOS"
+    rune -0 cscli scenarios list -o json
+    assert_json '{scenarios:[]}'
+    rune -0 cscli scenarios list -o raw
+    assert_output 'name,status,version,description'
+
+    # some items
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf
+
+    rune -0 cscli scenarios list
+    assert_output --partial crowdsecurity/ssh-bf
+    assert_output --partial crowdsecurity/telnet-bf
+    rune -0 grep -c enabled <(output)
+    assert_output "2"
+
+    rune -0 cscli scenarios list -o json
+    assert_output --partial crowdsecurity/ssh-bf
+    assert_output --partial crowdsecurity/telnet-bf
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output "2"
+
+    rune -0 cscli scenarios list -o raw
+    assert_output --partial crowdsecurity/ssh-bf
+    assert_output --partial crowdsecurity/telnet-bf
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+}
+
+@test "cscli scenarios list -a" {
+    expected=$(jq <"$HUB_DIR/.index.json" -r '.scenarios | length')
+
+    rune -0 cscli scenarios list -a
+    rune -0 grep -c disabled <(output)
+    assert_output "$expected"
+
+    rune -0 cscli scenarios list -o json -a
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output "$expected"
+
+    rune -0 cscli scenarios list -o raw -a
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "$expected"
+}
+
+@test "cscli scenarios list [scenario]..." {
+    # non-existent
+    rune -1 cscli scenario install foo/bar
+    assert_stderr --partial "can't find 'foo/bar' in scenarios"
+ 
+    # not installed
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf
+    assert_output --regexp 'crowdsecurity/ssh-bf.*disabled'
+
+    # install two items
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf
+
+    # list an installed item
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf
+    assert_output --regexp "crowdsecurity/ssh-bf.*enabled"
+    refute_output --partial "crowdsecurity/telnet-bf"
+
+    # list multiple installed and non installed items
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf
+    assert_output --partial "crowdsecurity/ssh-bf"
+    assert_output --partial "crowdsecurity/telnet-bf"
+    assert_output --partial "crowdsecurity/aws-bf"
+
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output "1"
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json
+    rune -0 jq '.scenarios | length' <(output)
+    assert_output "3"
+
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "1"
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "3"
+}
+
+@test "cscli scenarios install [scenario]..." {
+    rune -1 cscli scenarios install
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+
+    # not in hub
+    rune -1 cscli scenarios install crowdsecurity/blahblah
+    assert_stderr --partial "can't find 'crowdsecurity/blahblah' in scenarios"
+
+    # simple install
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics
+    assert_output --partial 'crowdsecurity/ssh-bf'
+    assert_output --partial 'installed: true'
+
+    # autocorrect
+    rune -1 cscli scenarios install crowdsecurity/ssh-tf
+    assert_stderr --partial "can't find 'crowdsecurity/ssh-tf' in scenarios, did you mean crowdsecurity/ssh-bf?"
+
+    # install multiple
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics
+    assert_output --partial 'crowdsecurity/ssh-bf'
+    assert_output --partial 'installed: true'
+    rune -0 cscli scenarios inspect crowdsecurity/telnet-bf --no-metrics
+    assert_output --partial 'crowdsecurity/telnet-bf'
+    assert_output --partial 'installed: true'
+}
+
+
+@test "cscli scenarios install [scenario]... (file location and download-only)" {
+    # simple install
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics
+    assert_output --partial 'crowdsecurity/ssh-bf'
+    assert_output --partial 'installed: false'
+    assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml"
+    assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
+
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
+    assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
+}
+
+
+@test "cscli scenarios inspect [scenario]..." {
+    rune -1 cscli scenarios inspect
+    assert_stderr --partial 'requires at least 1 arg(s), only received 0'
+    ./instance-crowdsec start
+
+    rune -1 cscli scenarios inspect blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios"
+
+    # one item
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics
+    assert_line 'type: scenarios'
+    assert_line 'name: crowdsecurity/ssh-bf'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # one item, with metrics
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf
+    assert_line --partial 'Current metrics:'
+
+    # one item, json
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json
+    rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output)
+    # XXX: .installed is missing -- not false
+    assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null]'
+
+    # one item, raw
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o raw
+    assert_line 'type: scenarios'
+    assert_line 'name: crowdsecurity/ssh-bf'
+    assert_line 'author: crowdsecurity'
+    assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml'
+    assert_line 'installed: false'
+    refute_line --partial 'Current metrics:'
+
+    # multiple items
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf --no-metrics
+    assert_output --partial 'crowdsecurity/ssh-bf'
+    assert_output --partial 'crowdsecurity/telnet-bf'
+    rune -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+
+    # multiple items, with metrics
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf
+    rune -0 grep -c 'Current metrics:' <(output)
+    assert_output "2"
+
+    # multiple items, json
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json
+    rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output)
+    assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",null]]'
+
+    # multiple items, raw
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw
+    assert_output --partial 'crowdsecurity/ssh-bf'
+    assert_output --partial 'crowdsecurity/telnet-bf'
+    run -1 grep -c 'Current metrics:' <(output)
+    assert_output "0"
+}
+
+@test "cscli scenarios remove [scenario]..." {
+    rune -1 cscli scenarios remove
+    assert_stderr --partial "specify at least one scenario to remove or '--all'"
+
+    rune -1 cscli scenarios remove blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios"
+
+    # XXX: we can however remove a real item if it's not installed, or already removed
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf
+
+    # install, then remove, check files
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
+    assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf
+    assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
+
+    # delete is an alias for remove
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
+    assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
+    rune -0 cscli scenarios delete crowdsecurity/ssh-bf
+    assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
+
+    # purge
+    assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml"
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge
+    assert_file_not_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml"
+
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf
+
+    # --all
+    rune -0 cscli scenarios list -o raw
+    rune -0 grep -vc 'name,status,version,description' <(output)
+    assert_output "2"
+
+    rune -0 cscli scenarios remove --all
+
+    rune -0 cscli scenarios list -o raw
+    rune -1 grep -vc 'name,status,version,description' <(output)
+    assert_output "0"
+}
+
+@test "cscli scenarios upgrade [scenario]..." {
+    rune -1 cscli scenarios upgrade
+    assert_stderr --partial "specify at least one scenario to upgrade or '--all'"
+
+    # XXX: should this return 1 instead of log.Error?
+    rune -0 cscli scenarios upgrade blahblah/blahblah
+    assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios"
+
+    # XXX: same message if the item exists but is not installed, this is confusing
+    rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf
+    assert_stderr --partial "can't find 'crowdsecurity/ssh-bf' in scenarios"
+
+    # hash of an empty file
+    sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+    # add version 0.0 to the hub
+    new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {scenarios:{"crowdsecurity/ssh-bf":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
+    echo "$new_hub" >"$HUB_DIR/.index.json"
+ 
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
+
+    # bring the file to v0.0
+    truncate -s 0 "$CONFIG_DIR/scenarios/ssh-bf.yaml"
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json
+    rune -0 jq -e '.local_version=="0.0"' <(output)
+
+    # upgrade
+    rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # taint
+    echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
+    # XXX: should return error
+    rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf
+    assert_stderr --partial "crowdsecurity/ssh-bf is tainted, --force to overwrite"
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json
+    rune -0 jq -e '.local_version=="?"' <(output)
+
+    # force upgrade with taint
+    rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf --force
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json
+    rune -0 jq -e '.local_version==.version' <(output)
+
+    # multiple items
+    rune -0 cscli scenarios install crowdsecurity/telnet-bf
+    echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
+    echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml"
+    rune -0 cscli scenarios list -o json
+    rune -0 jq -e '[.scenarios[].local_version]==["?","?"]' <(output)
+    rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/telnet-bf
+    rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output)
+
+    # upgrade all
+    echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
+    echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml"
+    rune -0 cscli scenarios upgrade --all
+    rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output)
+}

test/lib/setup_file.sh

@@ -238,6 +238,12 @@ assert_stderr_line() {
 }
 export -f assert_stderr_line
 
+hub_uninstall_all() {
+    CONFIG_DIR=$(dirname "$CONFIG_YAML")
+    rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/*
+}
+export -f hub_uninstall_all
+
 # remove color and style sequences from stdin
 plaintext() {
     sed -E 's/\x1B\[[0-9;]*[JKmsu]//g'