Refact pkg/cwhub, cmd/crowdsec-cli (#2557)

 - pkg/cwhub: change file layout, rename functions
 - method Item.SubItems
 - cmd/crowdsec-cli: generic code for hub items
 - cscli: removing any type of items in a collection now requires --force
 - tests

cmd/crowdsec-cli/collections.go

@@ -1,333 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCollectionsCmd() *cobra.Command {
-	cmdCollections := &cobra.Command{
-		Use:   "collections <action> [collection]...",
-		Short: "Manage hub collections",
-		Example: `cscli collections list -a
-cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"collection"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if _, err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdCollections.AddCommand(NewCollectionsInstallCmd())
-	cmdCollections.AddCommand(NewCollectionsRemoveCmd())
-	cmdCollections.AddCommand(NewCollectionsUpgradeCmd())
-	cmdCollections.AddCommand(NewCollectionsInspectCmd())
-	cmdCollections.AddCommand(NewCollectionsListCmd())
-
-	return cmdCollections
-}
-
-func runCollectionsInstall(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	downloadOnly, err := flags.GetBool("download-only")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	ignoreError, err := flags.GetBool("ignore")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		t := hub.GetItem(cwhub.COLLECTIONS, name)
-		if t == nil {
-			nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
-			Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
-
-			continue
-		}
-
-		if err := hub.InstallItem(name, cwhub.COLLECTIONS, force, downloadOnly); err != nil {
-			if !ignoreError {
-				return fmt.Errorf("error while installing '%s': %w", name, err)
-			}
-			log.Errorf("Error while installing '%s': %s", name, err)
-		}
-	}
-
-	return nil
-}
-
-func NewCollectionsInstallCmd() *cobra.Command {
-	cmdCollectionsInstall := &cobra.Command{
-		Use:               "install <collection>...",
-		Short:             "Install given collection(s)",
-		Long:              `Fetch and install one or more collections from hub`,
-		Example:           `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: runCollectionsInstall,
-	}
-
-	flags := cmdCollectionsInstall.Flags()
-	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
-	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
-	flags.Bool("ignore", false, "Ignore errors when installing multiple collections")
-
-	return cmdCollectionsInstall
-}
-
-func runCollectionsRemove(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	purge, err := flags.GetBool("purge")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		err := hub.RemoveMany(cwhub.COLLECTIONS, "", all, purge, force)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one collection to remove or '--all'")
-	}
-
-	for _, name := range args {
-		if !force {
-			item := hub.GetItem(cwhub.COLLECTIONS, name)
-			if item == nil {
-				// XXX: this should be in GetItem?
-				return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS)
-			}
-			if len(item.BelongsToCollections) > 0 {
-				log.Warningf("%s belongs to other collections: %s", name, item.BelongsToCollections)
-				log.Warningf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection", name)
-				continue
-			}
-		}
-
-		err := hub.RemoveMany(cwhub.COLLECTIONS, name, all, purge, force)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewCollectionsRemoveCmd() *cobra.Command {
-	cmdCollectionsRemove := &cobra.Command{
-		Use:               "remove <collection>...",
-		Short:             "Remove given collection(s)",
-		Long:              `Remove one or more collections`,
-		Example:           `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: runCollectionsRemove,
-	}
-
-	flags := cmdCollectionsRemove.Flags()
-	flags.Bool("purge", false, "Delete source file too")
-	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
-	flags.Bool("all", false, "Remove all the collections")
-
-	return cmdCollectionsRemove
-}
-
-func runCollectionsUpgrade(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one collection to upgrade or '--all'")
-	}
-
-	for _, name := range args {
-		if err := hub.UpgradeConfig(cwhub.COLLECTIONS, name, force); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewCollectionsUpgradeCmd() *cobra.Command {
-	cmdCollectionsUpgrade := &cobra.Command{
-		Use:               "upgrade <collection>...",
-		Short:             "Upgrade given collection(s)",
-		Long:              `Fetch and upgrade one or more collections from the hub`,
-		Example:           `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: runCollectionsUpgrade,
-	}
-
-	flags := cmdCollectionsUpgrade.Flags()
-	flags.BoolP("all", "a", false, "Upgrade all the collections")
-	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
-
-	return cmdCollectionsUpgrade
-}
-
-func runCollectionsInspect(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	url, err := flags.GetString("url")
-	if err != nil {
-		return err
-	}
-
-	if url != "" {
-		csConfig.Cscli.PrometheusUrl = url
-	}
-
-	noMetrics, err := flags.GetBool("no-metrics")
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewCollectionsInspectCmd() *cobra.Command {
-	cmdCollectionsInspect := &cobra.Command{
-		Use:               "inspect <collection>...",
-		Short:             "Inspect given collection(s)",
-		Long:              `Inspect one or more collections`,
-		Example:           `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: runCollectionsInspect,
-	}
-
-	flags := cmdCollectionsInspect.Flags()
-	flags.StringP("url", "u", "", "Prometheus url")
-	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
-
-	return cmdCollectionsInspect
-}
-
-func runCollectionsList(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func NewCollectionsListCmd() *cobra.Command {
-	cmdCollectionsList := &cobra.Command{
-		Use:   "list [collection... | -a]",
-		Short: "List collections",
-		Long:  `List of installed/available/specified collections`,
-		Example: `cscli collections list
-cscli collections list -a
-cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`,
-		DisableAutoGenTag: true,
-		RunE:              runCollectionsList,
-	}
-
-	flags := cmdCollectionsList.Flags()
-	flags.BoolP("all", "a", false, "List disabled items as well")
-
-	return cmdCollectionsList
-}

cmd/crowdsec-cli/config_restore.go

@@ -36,7 +36,7 @@ func silentInstallItem(name string, obtype string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
 	}
-	if err := hub.AddItem(obtype, *item); err != nil {
+	if err := hub.AddItem(*item); err != nil {
 		return "", err
 	}
 
@@ -44,7 +44,7 @@ func silentInstallItem(name string, obtype string) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
 	}
-	if err := hub.AddItem(obtype, *item); err != nil {
+	if err := hub.AddItem(*item); err != nil {
 		return "", err
 	}
 	return fmt.Sprintf("Enabled %s", item.Name), nil

cmd/crowdsec-cli/hub.go

@@ -61,7 +61,9 @@ func runHubList(cmd *cobra.Command, args []string) error {
 		log.Info(v)
 	}
 
-	cwhub.DisplaySummary()
+	for line := range hub.ItemStats() {
+		log.Info(line)
+	}
 
 	err = ListItems(color.Output, []string{
 		cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS,

cmd/crowdsec-cli/itemcommands.go

@@ -0,0 +1,503 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/go-cs-lib/coalesce"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+type cmdHelp struct {
+	// Example is required, the others have a default value
+	// generated from the item type
+	use     string
+	short   string
+	long    string
+	example string
+}
+
+type hubItemType struct {
+	name        string // plural, as used in the hub index
+	singular    string
+	oneOrMore   string // parenthetical pluralizaion: "parser(s)"
+	help        cmdHelp
+	installHelp cmdHelp
+	removeHelp  cmdHelp
+	upgradeHelp cmdHelp
+	inspectHelp cmdHelp
+	listHelp    cmdHelp
+}
+
+var hubItemTypes = map[string]hubItemType{
+	"parsers": {
+		name:      "parsers",
+		singular:  "parser",
+		oneOrMore: "parser(s)",
+		help: cmdHelp{
+			example: `cscli parsers list -a
+cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli parsers list
+cscli parsers list -a
+cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+	},
+	"postoverflows": {
+		name:      "postoverflows",
+		singular:  "postoverflow",
+		oneOrMore: "postoverflow(s)",
+		help: cmdHelp{
+			example: `cscli postoverflows list -a
+cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli postoverflows list
+cscli postoverflows list -a
+cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+	},
+	"scenarios": {
+		name:      "scenarios",
+		singular:  "scenario",
+		oneOrMore: "scenario(s)",
+		help: cmdHelp{
+			example: `cscli scenarios list -a
+cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli scenarios list
+cscli scenarios list -a
+cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+	},
+	"collections": {
+		name:      "collections",
+		singular:  "collection",
+		oneOrMore: "collection(s)",
+		help: cmdHelp{
+			example: `cscli collections list -a
+cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
+`,
+		},
+		installHelp: cmdHelp{
+			example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		removeHelp: cmdHelp{
+			example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		upgradeHelp: cmdHelp{
+			example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		inspectHelp: cmdHelp{
+			example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		listHelp: cmdHelp{
+			example: `cscli collections list
+cscli collections list -a
+cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+	},
+}
+
+func NewItemsCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
+		Short:             coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
+		Long:              it.help.long,
+		Example:           it.help.example,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{it.singular},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if _, err := require.Hub(csConfig); err != nil {
+				return err
+			}
+
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	cmd.AddCommand(NewItemsInstallCmd(typeName))
+	cmd.AddCommand(NewItemsRemoveCmd(typeName))
+	cmd.AddCommand(NewItemsUpgradeCmd(typeName))
+	cmd.AddCommand(NewItemsInspectCmd(typeName))
+	cmd.AddCommand(NewItemsListCmd(typeName))
+
+	return cmd
+}
+
+func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		downloadOnly, err := flags.GetBool("download-only")
+		if err != nil {
+			return err
+		}
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		ignoreError, err := flags.GetBool("ignore")
+		if err != nil {
+			return err
+		}
+
+		hub, err := cwhub.GetHub()
+		if err != nil {
+			return err
+		}
+
+		for _, name := range args {
+			t := hub.GetItem(it.name, name)
+			if t == nil {
+				nearestItem, score := GetDistance(it.name, name)
+				Suggest(it.name, name, nearestItem.Name, score, ignoreError)
+
+				continue
+			}
+
+			if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil {
+				if !ignoreError {
+					return fmt.Errorf("error while installing '%s': %w", name, err)
+				}
+				log.Errorf("Error while installing '%s': %s", name, err)
+			}
+		}
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsInstallCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.installHelp.use, "install [item]..."),
+		Short:             coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
+		Example:           it.installHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(typeName, args, toComplete)
+		},
+		RunE: itemsInstallRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
+
+	return cmd
+}
+
+func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		purge, err := flags.GetBool("purge")
+		if err != nil {
+			return err
+		}
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := cwhub.GetHub()
+		if err != nil {
+			return err
+		}
+
+		if all {
+			err := hub.RemoveMany(it.name, "", all, purge, force)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}
+
+		if len(args) == 0 {
+			return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
+		}
+
+		for _, name := range args {
+			if !force {
+				item := hub.GetItem(it.name, name)
+				if item == nil {
+					// XXX: this should be in GetItem?
+					return fmt.Errorf("can't find '%s' in %s", name, it.name)
+				}
+				if len(item.BelongsToCollections) > 0 {
+					log.Warningf("%s belongs to collections: %s", name, item.BelongsToCollections)
+					log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", it.name, name, it.singular)
+					continue
+				}
+			}
+
+			err := hub.RemoveMany(it.name, name, all, purge, force)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+	return run
+}
+
+func NewItemsRemoveCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.removeHelp.use, "remove [item]..."),
+		Short:             coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
+		Example:           it.removeHelp.example,
+		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsRemoveRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
+
+	return cmd
+}
+
+func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		force, err := flags.GetBool("force")
+		if err != nil {
+			return err
+		}
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		hub, err := cwhub.GetHub()
+		if err != nil {
+			return err
+		}
+
+		if all {
+			if err := hub.UpgradeConfig(it.name, "", force); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		if len(args) == 0 {
+			return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
+		}
+
+		for _, name := range args {
+			if err := hub.UpgradeConfig(it.name, name, force); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsUpgradeCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
+		Short:             coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
+		Example:           it.upgradeHelp.example,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsUpgradeRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmd
+}
+
+func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		url, err := flags.GetString("url")
+		if err != nil {
+			return err
+		}
+
+		if url != "" {
+			csConfig.Cscli.PrometheusUrl = url
+		}
+
+		noMetrics, err := flags.GetBool("no-metrics")
+		if err != nil {
+			return err
+		}
+
+		for _, name := range args {
+			if err = InspectItem(name, it.name, noMetrics); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsInspectCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.inspectHelp.use, "inspect [item]..."),
+		Short:             coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
+		Long:              coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
+		Example:           it.inspectHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(it.name, args, toComplete)
+		},
+		RunE: itemsInspectRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
+
+	return cmd
+}
+
+func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
+	run := func(cmd *cobra.Command, args []string) error {
+		flags := cmd.Flags()
+
+		all, err := flags.GetBool("all")
+		if err != nil {
+			return err
+		}
+
+		if err = ListItems(color.Output, []string{it.name}, args, false, true, all); err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	return run
+}
+
+func NewItemsListCmd(typeName string) *cobra.Command {
+	it := hubItemTypes[typeName]
+
+	cmd := &cobra.Command{
+		Use:               coalesce.String(it.listHelp.use, "list [item... | -a]"),
+		Short:             coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
+		Long:              coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
+		Example:           it.listHelp.example,
+		DisableAutoGenTag: true,
+		RunE:              itemsListRunner(it),
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmd
+}

cmd/crowdsec-cli/main.go

@@ -234,10 +234,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewSimulationCmds())
 	rootCmd.AddCommand(NewBouncersCmd())
 	rootCmd.AddCommand(NewMachinesCmd())
-	rootCmd.AddCommand(NewParsersCmd())
-	rootCmd.AddCommand(NewScenariosCmd())
-	rootCmd.AddCommand(NewCollectionsCmd())
-	rootCmd.AddCommand(NewPostOverflowsCmd())
 	rootCmd.AddCommand(NewCapiCmd())
 	rootCmd.AddCommand(NewLapiCmd())
 	rootCmd.AddCommand(NewCompletionCmd())
@@ -246,6 +242,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	rootCmd.AddCommand(NewHubTestCmd())
 	rootCmd.AddCommand(NewNotificationsCmd())
 	rootCmd.AddCommand(NewSupportCmd())
+	rootCmd.AddCommand(NewItemsCmd("collections"))
+	rootCmd.AddCommand(NewItemsCmd("parsers"))
+	rootCmd.AddCommand(NewItemsCmd("scenarios"))
+	rootCmd.AddCommand(NewItemsCmd("postoverflows"))
 
 	if fflag.CscliSetup.IsEnabled() {
 		rootCmd.AddCommand(NewSetupCmd())

cmd/crowdsec-cli/parsers.go

@@ -1,320 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewParsersCmd() *cobra.Command {
-	cmdParsers := &cobra.Command{
-		Use:   "parsers <action> [parser]...",
-		Short: "Manage hub parsers",
-		Example: `cscli parsers list -a
-cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"parser"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if _, err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdParsers.AddCommand(NewParsersInstallCmd())
-	cmdParsers.AddCommand(NewParsersRemoveCmd())
-	cmdParsers.AddCommand(NewParsersUpgradeCmd())
-	cmdParsers.AddCommand(NewParsersInspectCmd())
-	cmdParsers.AddCommand(NewParsersListCmd())
-
-	return cmdParsers
-}
-
-func runParsersInstall(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	downloadOnly, err := flags.GetBool("download-only")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	ignoreError, err := flags.GetBool("ignore")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		t := hub.GetItem(cwhub.PARSERS, name)
-		if t == nil {
-			nearestItem, score := GetDistance(cwhub.PARSERS, name)
-			Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
-
-			continue
-		}
-
-		if err := hub.InstallItem(name, cwhub.PARSERS, force, downloadOnly); err != nil {
-			if !ignoreError {
-				return fmt.Errorf("error while installing '%s': %w", name, err)
-			}
-			log.Errorf("Error while installing '%s': %s", name, err)
-		}
-	}
-
-	return nil
-}
-
-func NewParsersInstallCmd() *cobra.Command {
-	cmdParsersInstall := &cobra.Command{
-		Use:               "install <parser>...",
-		Short:             "Install given parser(s)",
-		Long:              `Fetch and install one or more parsers from the hub`,
-		Example:           `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: runParsersInstall,
-	}
-
-	flags := cmdParsersInstall.Flags()
-	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
-	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
-	flags.Bool("ignore", false, "Ignore errors when installing multiple parsers")
-
-	return cmdParsersInstall
-}
-
-func runParsersRemove(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	purge, err := flags.GetBool("purge")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		err := hub.RemoveMany(cwhub.PARSERS, "", all, purge, force)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one parser to remove or '--all'")
-	}
-
-	for _, name := range args {
-		err := hub.RemoveMany(cwhub.PARSERS, name, all, purge, force)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewParsersRemoveCmd() *cobra.Command {
-	cmdParsersRemove := &cobra.Command{
-		Use:               "remove <parser>...",
-		Short:             "Remove given parser(s)",
-		Long:              `Remove one or more parsers`,
-		Example:           `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: runParsersRemove,
-	}
-
-	flags := cmdParsersRemove.Flags()
-	flags.Bool("purge", false, "Delete source file too")
-	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
-	flags.Bool("all", false, "Remove all the parsers")
-
-	return cmdParsersRemove
-}
-
-func runParsersUpgrade(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one parser to upgrade or '--all'")
-	}
-
-	for _, name := range args {
-		if err := hub.UpgradeConfig(cwhub.PARSERS, name, force); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewParsersUpgradeCmd() *cobra.Command {
-	cmdParsersUpgrade := &cobra.Command{
-		Use:               "upgrade <parser>...",
-		Short:             "Upgrade given parser(s)",
-		Long:              `Fetch and upgrade one or more parsers from the hub`,
-		Example:           `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: runParsersUpgrade,
-	}
-
-	flags := cmdParsersUpgrade.Flags()
-	flags.BoolP("all", "a", false, "Upgrade all the parsers")
-	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
-
-	return cmdParsersUpgrade
-}
-
-func runParsersInspect(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	url, err := flags.GetString("url")
-	if err != nil {
-		return err
-	}
-
-	if url != "" {
-		csConfig.Cscli.PrometheusUrl = url
-	}
-
-	noMetrics, err := flags.GetBool("no-metrics")
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		if err = InspectItem(name, cwhub.PARSERS, noMetrics); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewParsersInspectCmd() *cobra.Command {
-	cmdParsersInspect := &cobra.Command{
-		Use:               "inspect <parser>",
-		Short:             "Inspect a parser",
-		Long:              `Inspect a parser`,
-		Example:           `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: runParsersInspect,
-	}
-
-	flags := cmdParsersInspect.Flags()
-	flags.StringP("url", "u", "", "Prometheus url")
-	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
-
-	return cmdParsersInspect
-}
-
-func runParsersList(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func NewParsersListCmd() *cobra.Command {
-	cmdParsersList := &cobra.Command{
-		Use:   "list [parser... | -a]",
-		Short: "List parsers",
-		Long:  `List of installed/available/specified parsers`,
-		Example: `cscli parsers list
-cscli parsers list -a
-cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		DisableAutoGenTag: true,
-		RunE:              runParsersList,
-	}
-
-	flags := cmdParsersList.Flags()
-	flags.BoolP("all", "a", false, "List disabled items as well")
-
-	return cmdParsersList
-}

cmd/crowdsec-cli/postoverflows.go

@@ -1,321 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewPostOverflowsCmd() *cobra.Command {
-	cmdPostOverflows := &cobra.Command{
-		Use:   "postoverflows <action> [postoverflow]...",
-		Short: "Manage hub postoverflows",
-		Example: `cscli postoverflows list -a
-cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"postoverflow"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if _, err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
-
-	return cmdPostOverflows
-}
-
-func runPostOverflowsInstall(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	downloadOnly, err := flags.GetBool("download-only")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	ignoreError, err := flags.GetBool("ignore")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		t := hub.GetItem(cwhub.POSTOVERFLOWS, name)
-		if t == nil {
-			nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name)
-			Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError)
-
-			continue
-		}
-
-		if err := hub.InstallItem(name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil {
-			if !ignoreError {
-				return fmt.Errorf("error while installing '%s': %w", name, err)
-			}
-			log.Errorf("Error while installing '%s': %s", name, err)
-		}
-	}
-
-	return nil
-}
-
-func NewPostOverflowsInstallCmd() *cobra.Command {
-	cmdPostOverflowsInstall := &cobra.Command{
-		Use:               "install <postoverflow>...",
-		Short:             "Install given postoverflow(s)",
-		Long:              `Fetch and install one or more postoverflows from the hub`,
-		Example:           `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.POSTOVERFLOWS, args, toComplete)
-		},
-		RunE: runPostOverflowsInstall,
-	}
-
-	flags := cmdPostOverflowsInstall.Flags()
-	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
-	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
-	flags.Bool("ignore", false, "Ignore errors when installing multiple postoverflows")
-
-	return cmdPostOverflowsInstall
-}
-
-func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	purge, err := flags.GetBool("purge")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		err := hub.RemoveMany(cwhub.POSTOVERFLOWS, "", all, purge, force)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
-	}
-
-	for _, name := range args {
-		err := hub.RemoveMany(cwhub.POSTOVERFLOWS, name, all, purge, force)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewPostOverflowsRemoveCmd() *cobra.Command {
-	cmdPostOverflowsRemove := &cobra.Command{
-		Use:               "remove <postoverflow>...",
-		Short:             "Remove given postoverflow(s)",
-		Long:              `remove one or more postoverflows from the hub`,
-		Example:           `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
-		},
-		RunE: runPostOverflowsRemove,
-	}
-
-	flags := cmdPostOverflowsRemove.Flags()
-	flags.Bool("purge", false, "Delete source file too")
-	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
-	flags.Bool("all", false, "Delete all the postoverflows")
-
-	return cmdPostOverflowsRemove
-}
-
-func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
-	}
-
-	for _, name := range args {
-		if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, name, force); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewPostOverflowsUpgradeCmd() *cobra.Command {
-	cmdPostOverflowsUpgrade := &cobra.Command{
-		Use:               "upgrade <postoverflow>...",
-		Short:             "Upgrade given postoverflow(s)",
-		Long:              `Fetch and upgrade one or more postoverflows from the hub`,
-		Example:           `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
-		},
-		RunE: runPostOverflowUpgrade,
-	}
-
-	flags := cmdPostOverflowsUpgrade.Flags()
-	flags.BoolP("all", "a", false, "Upgrade all the postoverflows")
-	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
-
-	return cmdPostOverflowsUpgrade
-}
-
-func runPostOverflowsInspect(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	url, err := flags.GetString("url")
-	if err != nil {
-		return err
-	}
-
-	if url != "" {
-		csConfig.Cscli.PrometheusUrl = url
-	}
-
-	noMetrics, err := flags.GetBool("no-metrics")
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		if err = InspectItem(name, cwhub.POSTOVERFLOWS, noMetrics); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewPostOverflowsInspectCmd() *cobra.Command {
-	cmdPostOverflowsInspect := &cobra.Command{
-		Use:               "inspect <postoverflow>",
-		Short:             "Inspect a postoverflow",
-		Long:              `Inspect a postoverflow`,
-		Example:           `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
-		},
-		RunE: runPostOverflowsInspect,
-	}
-
-	flags := cmdPostOverflowsInspect.Flags()
-
-	flags.StringP("url", "u", "", "Prometheus url")
-	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
-
-	return cmdPostOverflowsInspect
-}
-
-func runPostOverflowsList(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	if err = ListItems(color.Output, []string{cwhub.POSTOVERFLOWS}, args, false, true, all); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func NewPostOverflowsListCmd() *cobra.Command {
-	cmdPostOverflowsList := &cobra.Command{
-		Use:   "list [postoverflow]...",
-		Short: "List postoverflows",
-		Long:  `List of installed/available/specified postoverflows`,
-		Example: `cscli postoverflows list
-cscli postoverflows list -a
-cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		DisableAutoGenTag: true,
-		RunE:              runPostOverflowsList,
-	}
-
-	flags := cmdPostOverflowsList.Flags()
-	flags.BoolP("all", "a", false, "List disabled items as well")
-
-	return cmdPostOverflowsList
-}

cmd/crowdsec-cli/scenarios.go

@@ -1,320 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewScenariosCmd() *cobra.Command {
-	cmdScenarios := &cobra.Command{
-		Use:   "scenarios <action> [scenario]...",
-		Short: "Manage hub scenarios",
-		Example: `cscli scenarios list -a
-cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"scenario"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if _, err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdScenarios.AddCommand(NewCmdScenariosInstall())
-	cmdScenarios.AddCommand(NewCmdScenariosRemove())
-	cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
-	cmdScenarios.AddCommand(NewCmdScenariosInspect())
-	cmdScenarios.AddCommand(NewCmdScenariosList())
-
-	return cmdScenarios
-}
-
-func runScenariosInstall(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	downloadOnly, err := flags.GetBool("download-only")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	ignoreError, err := flags.GetBool("ignore")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		t := hub.GetItem(cwhub.SCENARIOS, name)
-		if t == nil {
-			nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
-			Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
-
-			continue
-		}
-
-		if err := hub.InstallItem(name, cwhub.SCENARIOS, force, downloadOnly); err != nil {
-			if !ignoreError {
-				return fmt.Errorf("error while installing '%s': %w", name, err)
-			}
-			log.Errorf("Error while installing '%s': %s", name, err)
-		}
-	}
-
-	return nil
-}
-
-func NewCmdScenariosInstall() *cobra.Command {
-	cmdScenariosInstall := &cobra.Command{
-		Use:               "install <scenario>...",
-		Short:             "Install given scenario(s)",
-		Long:              `Fetch and install one or more scenarios from the hub`,
-		Example:           `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		RunE: runScenariosInstall,
-	}
-
-	flags := cmdScenariosInstall.Flags()
-	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
-	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
-	flags.Bool("ignore", false, "Ignore errors when installing multiple scenarios")
-
-	return cmdScenariosInstall
-}
-
-func runScenariosRemove(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	purge, err := flags.GetBool("purge")
-	if err != nil {
-		return err
-	}
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		err := hub.RemoveMany(cwhub.SCENARIOS, "", all, purge, force)
-		if err != nil {
-			return err
-		}
-
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one scenario to remove or '--all'")
-	}
-
-	for _, name := range args {
-		err := hub.RemoveMany(cwhub.SCENARIOS, name, all, purge, force)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewCmdScenariosRemove() *cobra.Command {
-	cmdScenariosRemove := &cobra.Command{
-		Use:               "remove <scenario>...",
-		Short:             "Remove given scenario(s)",
-		Long:              `remove one or more scenarios`,
-		Example:           `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		RunE: runScenariosRemove,
-	}
-
-	flags := cmdScenariosRemove.Flags()
-	flags.Bool("purge", false, "Delete source file too")
-	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
-	flags.Bool("all", false, "Remove all the scenarios")
-
-	return cmdScenariosRemove
-}
-
-func runScenariosUpgrade(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	force, err := flags.GetBool("force")
-	if err != nil {
-		return err
-	}
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	hub, err := cwhub.GetHub()
-	if err != nil {
-		return err
-	}
-
-	if all {
-		if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil {
-			return err
-		}
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
-	}
-
-	for _, name := range args {
-		if err := hub.UpgradeConfig(cwhub.SCENARIOS, name, force); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewCmdScenariosUpgrade() *cobra.Command {
-	cmdScenariosUpgrade := &cobra.Command{
-		Use:               "upgrade <scenario>...",
-		Short:             "Upgrade given scenario(s)",
-		Long:              `Fetch and upgrade one or more scenarios from the hub`,
-		Example:           `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		RunE: runScenariosUpgrade,
-	}
-
-	flags := cmdScenariosUpgrade.Flags()
-	flags.BoolP("all", "a", false, "Upgrade all the scenarios")
-	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
-
-	return cmdScenariosUpgrade
-}
-
-func runScenariosInspect(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	url, err := flags.GetString("url")
-	if err != nil {
-		return err
-	}
-
-	if url != "" {
-		csConfig.Cscli.PrometheusUrl = url
-	}
-
-	noMetrics, err := flags.GetBool("no-metrics")
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		if err = InspectItem(name, cwhub.SCENARIOS, noMetrics); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func NewCmdScenariosInspect() *cobra.Command {
-	cmdScenariosInspect := &cobra.Command{
-		Use:               "inspect <scenario>",
-		Short:             "Inspect a scenario",
-		Long:              `Inspect a scenario`,
-		Example:           `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		RunE: runScenariosInspect,
-	}
-
-	flags := cmdScenariosInspect.Flags()
-	flags.StringP("url", "u", "", "Prometheus url")
-	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
-
-	return cmdScenariosInspect
-}
-
-func runScenariosList(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
-
-	all, err := flags.GetBool("all")
-	if err != nil {
-		return err
-	}
-
-	if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func NewCmdScenariosList() *cobra.Command {
-	cmdScenariosList := &cobra.Command{
-		Use:   "list [scenario]...",
-		Short: "List scenarios",
-		Long:  `List of installed/available/specified scenarios`,
-		Example: `cscli scenarios list
-cscli scenarios list -a
-cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		DisableAutoGenTag: true,
-		RunE:              runScenariosList,
-	}
-
-	flags := cmdScenariosList.Flags()
-	flags.BoolP("all", "a", false, "List disabled items as well")
-
-	return cmdScenariosList
-}

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/c-robinson/iplib v1.0.3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.4
+	github.com/crowdsecurity/go-cs-lib v0.0.5
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -137,8 +137,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
-github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
+github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
+github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

pkg/cwhub/branch.go

@@ -0,0 +1,59 @@
+package cwhub
+
+// Set the appropriate hub branch according to config settings and crowdsec version
+
+import (
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+)
+
+// chooseHubBranch returns the branch name to use for the hub
+// It can be "master" or the branch corresponding to the current crowdsec version
+func chooseHubBranch() string {
+	latest, err := cwversion.Latest()
+	if err != nil {
+		log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
+		return "master"
+	}
+
+	csVersion := cwversion.VersionStrip()
+	if csVersion == latest {
+		log.Debugf("current version is equal to latest (%s)", csVersion)
+		return "master"
+	}
+
+	// if current version is greater than the latest we are in pre-release
+	if semver.Compare(csVersion, latest) == 1 {
+		log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
+		return "master"
+	}
+
+	if csVersion == "" {
+		log.Warning("Crowdsec version is not set, using master branch for the hub")
+		return "master"
+	}
+
+	log.Warnf("Crowdsec is not the latest version. "+
+		"Current version is '%s' and the latest stable version is '%s'. Please update it!",
+		csVersion, latest)
+
+	log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
+		"added to Crowdsec Hub after CrowdSec %s", latest)
+
+	return csVersion
+}
+
+// SetHubBranch sets the package variable that points to the hub branch.
+func SetHubBranch() {
+	// a branch is already set, or specified from the flags
+	if HubBranch != "" {
+		return
+	}
+
+	// use the branch corresponding to the crowdsec version
+	HubBranch = chooseHubBranch()
+
+	log.Debugf("Using branch '%s' for the hub", HubBranch)
+}

pkg/cwhub/cwhub.go

@@ -5,14 +5,7 @@
 package cwhub
 
 import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/enescakir/emoji"
-	"github.com/pkg/errors"
-	"golang.org/x/mod/semver"
+	"errors"
 )
 
 var (
@@ -21,230 +14,3 @@ var (
 	RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
 	HubBranch          = "master"
 )
-
-// ItemVersion is used to detect the version of a given item
-// by comparing the hash of each version to the local file.
-// If the item does not match any known version, it is considered tainted.
-type ItemVersion struct {
-	Digest     string `json:"digest,omitempty"`     // meow
-	Deprecated bool   `json:"deprecated,omitempty"` // XXX: do we keep this?
-}
-
-// Item represents an object managed in the hub. It can be a parser, scenario, collection..
-type Item struct {
-	// descriptive info
-	Type                 string   `json:"type,omitempty"                   yaml:"type,omitempty"`                   // parser|postoverflows|scenario|collection(|enrich)
-	Stage                string   `json:"stage,omitempty"                  yaml:"stage,omitempty"`                  // Stage for parser|postoverflow: s00-raw/s01-...
-	Name                 string   `json:"name,omitempty"`                                                           // as seen in .index.json, usually "author/name"
-	FileName             string   `json:"file_name,omitempty"`                                                      // the filename, ie. apache2-logs.yaml
-	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .index.json
-	Author               string   `json:"author,omitempty"`                                                         // as seen in .index.json
-	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`             // as seen in .index.json
-	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
-
-	// remote (hub) info
-	RemotePath string                 `json:"path,omitempty"      yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
-	Version    string                 `json:"version,omitempty"`                                // the last version
-	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                     // the list of existing versions
-
-	// local (deployed) info
-	LocalPath    string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
-	LocalVersion string `json:"local_version,omitempty"`
-	LocalHash    string `json:"local_hash,omitempty"` // the local meow
-	Installed    bool   `json:"installed,omitempty"`
-	Downloaded   bool   `json:"downloaded,omitempty"`
-	UpToDate     bool   `json:"up_to_date,omitempty"`
-	Tainted      bool   `json:"tainted,omitempty"` // has it been locally modified
-	Local        bool   `json:"local,omitempty"`   // if it's a non versioned control one
-
-	// if it's a collection, it can have sub items
-	Parsers       []string `json:"parsers,omitempty"       yaml:"parsers,omitempty"`
-	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
-	Scenarios     []string `json:"scenarios,omitempty"     yaml:"scenarios,omitempty"`
-	Collections   []string `json:"collections,omitempty"   yaml:"collections,omitempty"`
-}
-
-// Status returns the status of the item as a string and an emoji
-// ie. "enabled,update-available" and emoji.Warning
-func (i *Item) Status() (string, emoji.Emoji) {
-	status := "disabled"
-	ok := false
-
-	if i.Installed {
-		ok = true
-		status = "enabled"
-	}
-
-	managed := true
-	if i.Local {
-		managed = false
-		status += ",local"
-	}
-
-	warning := false
-	if i.Tainted {
-		warning = true
-		status += ",tainted"
-	} else if !i.UpToDate && !i.Local {
-		warning = true
-		status += ",update-available"
-	}
-
-	emo := emoji.QuestionMark
-
-	switch {
-	case !managed:
-		emo = emoji.House
-	case !i.Installed:
-		emo = emoji.Prohibited
-	case warning:
-		emo = emoji.Warning
-	case ok:
-		emo = emoji.CheckMark
-	}
-
-	return status, emo
-}
-
-// versionStatus: semver requires 'v' prefix
-func (i *Item) versionStatus() int {
-	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
-}
-
-// validPath returns true if the (relative) path is allowed for the item
-// dirNmae: the directory name (ie. crowdsecurity)
-// fileName: the filename (ie. apache2-logs.yaml)
-func (i *Item) validPath(dirName, fileName string) bool {
-	return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
-}
-
-// GetItemMap returns the map of items for a given type
-func (h *Hub) GetItemMap(itemType string) map[string]Item {
-	m, ok := h.Items[itemType]
-	if !ok {
-		return nil
-	}
-
-	return m
-}
-
-// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
-// XXX: only used by leakybucket manager
-func itemKey(itemPath string) (string, error) {
-	f, err := os.Lstat(itemPath)
-	if err != nil {
-		return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
-	}
-
-	if f.Mode()&os.ModeSymlink == 0 {
-		// it's not a symlink, so the filename itsef should be the key
-		return filepath.Base(itemPath), nil
-	}
-
-	// resolve the symlink to hub file
-	pathInHub, err := os.Readlink(itemPath)
-	if err != nil {
-		return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
-	}
-
-	author := filepath.Base(filepath.Dir(pathInHub))
-
-	fname := filepath.Base(pathInHub)
-	fname = strings.TrimSuffix(fname, ".yaml")
-	fname = strings.TrimSuffix(fname, ".yml")
-
-	return fmt.Sprintf("%s/%s", author, fname), nil
-}
-
-// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
-func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
-	itemKey, err := itemKey(itemPath)
-	if err != nil {
-		return nil, err
-	}
-
-	m := h.GetItemMap(itemType)
-	if m == nil {
-		return nil, fmt.Errorf("item type %s doesn't exist", itemType)
-	}
-
-	v, ok := m[itemKey]
-	if !ok {
-		return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
-	}
-
-	return &v, nil
-}
-
-// GetItem returns the item from hub based on its type and full name (author/name)
-func (h *Hub) GetItem(itemType string, itemName string) *Item {
-	m, ok := h.GetItemMap(itemType)[itemName]
-	if !ok {
-		return nil
-	}
-
-	return &m
-}
-
-// GetItemNames returns the list of item (full) names for a given type
-// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
-// The names can be used to retrieve the item with GetItem()
-func (h *Hub) GetItemNames(itemType string) []string {
-	m := h.GetItemMap(itemType)
-	if m == nil {
-		return nil
-	}
-
-	names := make([]string, 0, len(m))
-	for k := range m {
-		names = append(names, k)
-	}
-
-	return names
-}
-
-// AddItem adds an item to the hub index
-func (h *Hub) AddItem(itemType string, item Item) error {
-	for _, itype := range ItemTypes {
-		if itype == itemType {
-			h.Items[itemType][item.Name] = item
-			return nil
-		}
-	}
-
-	return fmt.Errorf("ItemType %s is unknown", itemType)
-}
-
-// GetInstalledItems returns the list of installed items
-func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) {
-	items, ok := h.Items[itemType]
-	if !ok {
-		return nil, fmt.Errorf("no %s in hubIdx", itemType)
-	}
-
-	retItems := make([]Item, 0)
-
-	for _, item := range items {
-		if item.Installed {
-			retItems = append(retItems, item)
-		}
-	}
-
-	return retItems, nil
-}
-
-// GetInstalledItemsAsString returns the names of the installed items
-func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
-	items, err := h.GetInstalledItems(itemType)
-	if err != nil {
-		return nil, err
-	}
-
-	retStr := make([]string, len(items))
-
-	for i, it := range items {
-		retStr[i] = it.Name
-	}
-
-	return retStr, nil
-}

pkg/cwhub/cwhub_test.go

@@ -9,11 +9,8 @@ import (
 	"testing"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/crowdsecurity/go-cs-lib/cstest"
-
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
@@ -28,81 +25,6 @@ import (
 
 var responseByPath map[string]string
 
-func TestItemStatus(t *testing.T) {
-	hub := envSetup(t)
-
-	// get existing map
-	x := hub.GetItemMap(COLLECTIONS)
-	require.NotEmpty(t, x)
-
-	// Get item : good and bad
-	for k := range x {
-		item := hub.GetItem(COLLECTIONS, k)
-		require.NotNil(t, item)
-
-		item.Installed = true
-		item.UpToDate = false
-		item.Local = false
-		item.Tainted = false
-
-		txt, _ := item.Status()
-		require.Equal(t, "enabled,update-available", txt)
-
-		item.Installed = false
-		item.UpToDate = false
-		item.Local = true
-		item.Tainted = false
-
-		txt, _ = item.Status()
-		require.Equal(t, "disabled,local", txt)
-	}
-
-	err := DisplaySummary()
-	require.NoError(t, err)
-}
-
-func TestGetters(t *testing.T) {
-	hub := envSetup(t)
-
-	// get non existing map
-	empty := hub.GetItemMap("ratata")
-	require.Nil(t, empty)
-
-	// get existing map
-	x := hub.GetItemMap(COLLECTIONS)
-	require.NotEmpty(t, x)
-
-	// Get item : good and bad
-	for k := range x {
-		empty := hub.GetItem(COLLECTIONS, k+"nope")
-		require.Nil(t, empty)
-
-		item := hub.GetItem(COLLECTIONS, k)
-		require.NotNil(t, item)
-
-		// Add item and get it
-		item.Name += "nope"
-		err := hub.AddItem(COLLECTIONS, *item)
-		require.NoError(t, err)
-
-		newitem := hub.GetItem(COLLECTIONS, item.Name)
-		require.NotNil(t, newitem)
-
-		err = hub.AddItem("ratata", *item)
-		cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
-	}
-}
-
-func TestIndexDownload(t *testing.T) {
-	hub := envSetup(t)
-
-	_, err := InitHubUpdate(hub.cfg)
-	require.NoError(t, err, "failed to download index")
-
-	_, err = GetHub()
-	require.NoError(t, err, "failed to load hub index")
-}
-
 // testHub initializes a temporary hub with an empty json file, optionally updating it
 func testHub(t *testing.T, update bool) *Hub {
 	tmpDir, err := os.MkdirTemp("", "testhub")
@@ -115,13 +37,13 @@ func testHub(t *testing.T, update bool) *Hub {
 		InstallDataDir: filepath.Join(tmpDir, "installed-data"),
 	}
 
-	err = os.MkdirAll(hubCfg.HubDir, 0700)
+	err = os.MkdirAll(hubCfg.HubDir, 0o700)
 	require.NoError(t, err)
 
-	err = os.MkdirAll(hubCfg.InstallDir, 0700)
+	err = os.MkdirAll(hubCfg.InstallDir, 0o700)
 	require.NoError(t, err)
 
-	err = os.MkdirAll(hubCfg.InstallDataDir, 0700)
+	err = os.MkdirAll(hubCfg.InstallDataDir, 0o700)
 	require.NoError(t, err)
 
 	index, err := os.Create(hubCfg.HubIndexFile)
@@ -148,8 +70,9 @@ func testHub(t *testing.T, update bool) *Hub {
 	return hub
 }
 
+// envSetup initializes the temporary hub and mocks the http client
 func envSetup(t *testing.T) *Hub {
-	resetResponseByPath()
+	setResponseByPath()
 	log.SetLevel(log.DebugLevel)
 
 	defaultTransport := http.DefaultClient.Transport
@@ -163,151 +86,9 @@ func envSetup(t *testing.T) *Hub {
 
 	hub := testHub(t, true)
 
-	// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
-	// 	log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
-	// }
-	// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
-	// 	log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
-	// }
 	return hub
 }
 
-func testInstallItem(hub *Hub, t *testing.T, item Item) {
-	// Install the parser
-
-	err := hub.DownloadLatest(&item, false, false)
-	require.NoError(t, err, "failed to download %s", item.Name)
-
-	_, err = hub.LocalSync()
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
-
-	err = hub.EnableItem(&item)
-	require.NoError(t, err, "failed to enable %s", item.Name)
-
-	_, err = hub.LocalSync()
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
-}
-
-func testTaintItem(hub *Hub, t *testing.T, item Item) {
-	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
-
-	f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
-	require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
-
-	defer f.Close()
-
-	_, err = f.WriteString("tainted")
-	require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
-
-	// Local sync and check status
-	_, err = hub.LocalSync()
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
-}
-
-func testUpdateItem(hub *Hub, t *testing.T, item Item) {
-	assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
-
-	// Update it + check status
-	err := hub.DownloadLatest(&item, true, true)
-	require.NoError(t, err, "failed to update %s", item.Name)
-
-	// Local sync and check status
-	_, err = hub.LocalSync()
-	require.NoError(t, err, "failed to run localSync")
-
-	assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
-}
-
-func testDisableItem(hub *Hub, t *testing.T, item Item) {
-	assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
-
-	// Remove
-	err := hub.DisableItem(&item, false, false)
-	require.NoError(t, err, "failed to disable %s", item.Name)
-
-	// Local sync and check status
-	warns, err := hub.LocalSync()
-	require.NoError(t, err, "failed to run localSync")
-	require.Empty(t, warns, "unexpected warnings : %+v", warns)
-
-	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
-
-	// Purge
-	err = hub.DisableItem(&item, true, false)
-	require.NoError(t, err, "failed to purge %s", item.Name)
-
-	// Local sync and check status
-	warns, err = hub.LocalSync()
-	require.NoError(t, err, "failed to run localSync")
-	require.Empty(t, warns, "unexpected warnings : %+v", warns)
-
-	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
-}
-
-func TestInstallParser(t *testing.T) {
-	/*
-	 - install a random parser
-	 - check its status
-	 - taint it
-	 - check its status
-	 - force update it
-	 - check its status
-	 - remove it
-	*/
-	hub := envSetup(t)
-
-	// map iteration is random by itself
-	for _, it := range hub.Items[PARSERS] {
-		testInstallItem(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
-		testTaintItem(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
-		testUpdateItem(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
-		testDisableItem(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
-
-		break
-	}
-}
-
-func TestInstallCollection(t *testing.T) {
-	/*
-	 - install a random parser
-	 - check its status
-	 - taint it
-	 - check its status
-	 - force update it
-	 - check its status
-	 - remove it
-	*/
-	hub := envSetup(t)
-
-	// map iteration is random by itself
-	for _, it := range hub.Items[COLLECTIONS] {
-		testInstallItem(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
-		testTaintItem(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
-		testUpdateItem(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
-		testDisableItem(hub, t, it)
-		break
-	}
-}
-
 type mockTransport struct{}
 
 func newMockTransport() http.RoundTripper {
@@ -352,7 +133,7 @@ func fileToStringX(path string) string {
 	return strings.ReplaceAll(string(data), "\r\n", "\n")
 }
 
-func resetResponseByPath() {
+func setResponseByPath() {
 	responseByPath = map[string]string{
 		"/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml":    fileToStringX("./testdata/foobar_parser.yaml"),
 		"/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),

pkg/cwhub/dataset.go

@@ -1,6 +1,7 @@
 package cwhub
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -8,6 +9,7 @@ import (
 	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
@@ -39,7 +41,7 @@ func downloadFile(url string, destPath string) error {
 		return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body))
 	}
 
-	file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
 	if err != nil {
 		return err
 	}
@@ -70,3 +72,40 @@ func GetData(data []*types.DataSource, dataDir string) error {
 
 	return nil
 }
+
+// downloadData downloads the data files for an item
+func downloadData(dataFolder string, force bool, reader io.Reader) error {
+	var err error
+
+	dec := yaml.NewDecoder(reader)
+
+	for {
+		data := &DataSet{}
+
+		err = dec.Decode(data)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+
+			return fmt.Errorf("while reading file: %w", err)
+		}
+
+		download := false
+
+		for _, dataS := range data.Data {
+			if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
+				download = true
+			}
+		}
+
+		if download || force {
+			err = GetData(data.Data, dataFolder)
+			if err != nil {
+				return fmt.Errorf("while getting data: %w", err)
+			}
+		}
+	}
+
+	return nil
+}

pkg/cwhub/dataset_test.go

@@ -14,12 +14,14 @@ func TestDownloadFile(t *testing.T) {
 
 	httpmock.Activate()
 	defer httpmock.DeactivateAndReset()
+
 	//OK
 	httpmock.RegisterResponder(
 		"GET",
 		"https://example.com/xx",
 		httpmock.NewStringResponder(200, "example content oneoneone"),
 	)
+
 	httpmock.RegisterResponder(
 		"GET",
 		"https://example.com/x",
@@ -28,15 +30,19 @@ func TestDownloadFile(t *testing.T) {
 
 	err := downloadFile("https://example.com/xx", examplePath)
 	assert.NoError(t, err)
+
 	content, err := os.ReadFile(examplePath)
 	assert.Equal(t, "example content oneoneone", string(content))
 	assert.NoError(t, err)
+
 	//bad uri
 	err = downloadFile("https://zz.com", examplePath)
 	assert.Error(t, err)
+
 	//404
 	err = downloadFile("https://example.com/x", examplePath)
 	assert.Error(t, err)
+
 	//bad target
 	err = downloadFile("https://example.com/xx", "")
 	assert.Error(t, err)

pkg/cwhub/download.go

@@ -1,336 +0,0 @@
-package cwhub
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-)
-
-var ErrIndexNotFound = fmt.Errorf("index not found")
-
-// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk
-// It is used to inizialize the hub when there is no index file yet
-func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) {
-	if cfg == nil {
-		return nil, fmt.Errorf("no configuration found for hub")
-	}
-
-	bidx, err := DownloadHubIdx(cfg.HubIndexFile)
-	if err != nil {
-		return nil, fmt.Errorf("failed to download index: %w", err)
-	}
-
-	ret, err := ParseIndex(bidx)
-	if err != nil {
-		if !errors.Is(err, ErrMissingReference) {
-			return nil, fmt.Errorf("failed to read index: %w", err)
-		}
-	}
-
-	theHub = &Hub{
-		Items: ret,
-		cfg:   cfg,
-	}
-
-	if _, err := theHub.LocalSync(); err != nil {
-		return nil, fmt.Errorf("failed to sync: %w", err)
-	}
-
-	return theHub, nil
-}
-
-// DownloadHubIdx downloads the latest version of the index and returns the content
-func DownloadHubIdx(indexPath string) ([]byte, error) {
-	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
-
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to build request for hub index: %w", err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("failed http request for hub index: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		if resp.StatusCode == http.StatusNotFound {
-			return nil, ErrIndexNotFound
-		}
-
-		return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
-	}
-
-	oldContent, err := os.ReadFile(indexPath)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			log.Warningf("failed to read hub index: %s", err)
-		}
-	} else if bytes.Equal(body, oldContent) {
-		log.Info("hub index is up to date")
-		// write it anyway, can't hurt
-	}
-
-	file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-
-	if err != nil {
-		return nil, fmt.Errorf("while opening hub index file: %w", err)
-	}
-	defer file.Close()
-
-	wsize, err := file.Write(body)
-	if err != nil {
-		return nil, fmt.Errorf("while writing hub index file: %w", err)
-	}
-
-	log.Infof("Wrote new %d bytes index to %s", wsize, indexPath)
-
-	return body, nil
-}
-
-// DownloadLatest will download the latest version of Item to the tdir directory
-func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error {
-	var err error
-
-	log.Debugf("Downloading %s %s", target.Type, target.Name)
-
-	if target.Type != COLLECTIONS {
-		if !target.Installed && updateOnly && target.Downloaded {
-			log.Debugf("skipping upgrade of %s : not installed", target.Name)
-			return nil
-		}
-
-		return h.DownloadItem(target, overwrite)
-	}
-
-	// collection
-	var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
-	for idx, ptr := range tmp {
-		ptrtype := ItemTypes[idx]
-		for _, p := range ptr {
-			val, ok := h.Items[ptrtype][p]
-			if !ok {
-				return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
-			}
-
-			if !val.Installed && updateOnly && val.Downloaded {
-				log.Debugf("skipping upgrade of %s : not installed", target.Name)
-				continue
-			}
-
-			log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
-			//recurse as it's a collection
-			if ptrtype == COLLECTIONS {
-				log.Tracef("collection, recurse")
-
-				err = h.DownloadLatest(&val, overwrite, updateOnly)
-				if err != nil {
-					return fmt.Errorf("while downloading %s: %w", val.Name, err)
-				}
-			}
-
-			downloaded := val.Downloaded
-
-			err = h.DownloadItem(&val, overwrite)
-			if err != nil {
-				return fmt.Errorf("while downloading %s: %w", val.Name, err)
-			}
-
-			// We need to enable an item when it has been added to a collection since latest release of the collection.
-			// We check if val.Downloaded is false because maybe the item has been disabled by the user.
-			if !val.Installed && !downloaded {
-				if err = h.EnableItem(&val); err != nil {
-					return fmt.Errorf("enabling '%s': %w", val.Name, err)
-				}
-			}
-
-			h.Items[ptrtype][p] = val
-		}
-	}
-
-	err = h.DownloadItem(target, overwrite)
-	if err != nil {
-		return fmt.Errorf("failed to download item: %w", err)
-	}
-
-	return nil
-}
-
-func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
-	tdir := h.cfg.HubDir
-
-	// if user didn't --force, don't overwrite local, tainted, up-to-date files
-	if !overwrite {
-		if target.Tainted {
-			log.Debugf("%s : tainted, not updated", target.Name)
-			return nil
-		}
-
-		if target.UpToDate {
-			//  We still have to check if data files are present
-			log.Debugf("%s : up-to-date, not updated", target.Name)
-		}
-	}
-
-	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
-	}
-
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
-	}
-
-	hash := sha256.New()
-	if _, err = hash.Write(body); err != nil {
-		return fmt.Errorf("while hashing %s: %w", target.Name, err)
-	}
-
-	meow := fmt.Sprintf("%x", hash.Sum(nil))
-	if meow != target.Versions[target.Version].Digest {
-		log.Errorf("Downloaded version doesn't match index, please 'hub update'")
-		log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
-
-		return fmt.Errorf("invalid download hash for %s", target.Name)
-	}
-
-	//all good, install
-	//check if parent dir exists
-	tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
-	parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
-
-	// ensure that target file is within target dir
-	finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
-	if err != nil {
-		return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
-	}
-
-	if !strings.HasPrefix(finalPath, tdir) {
-		return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
-	}
-
-	// check dir
-	if _, err = os.Stat(parentDir); os.IsNotExist(err) {
-		log.Debugf("%s doesn't exist, create", parentDir)
-
-		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
-			return fmt.Errorf("while creating parent directories: %w", err)
-		}
-	}
-
-	// check actual file
-	if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
-		log.Warningf("%s : overwrite", target.Name)
-		log.Debugf("target: %s/%s", tdir, target.RemotePath)
-	} else {
-		log.Infof("%s : OK", target.Name)
-	}
-
-	f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
-	if err != nil {
-		return fmt.Errorf("while opening file: %w", err)
-	}
-
-	defer f.Close()
-
-	_, err = f.Write(body)
-	if err != nil {
-		return fmt.Errorf("while writing file: %w", err)
-	}
-
-	target.Downloaded = true
-	target.Tainted = false
-	target.UpToDate = true
-
-	if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
-		return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
-	}
-
-	h.Items[target.Type][target.Name] = *target
-
-	return nil
-}
-
-// DownloadDataIfNeeded downloads the data files for an item
-func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
-	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName)
-
-	itemFile, err := os.Open(itemFilePath)
-	if err != nil {
-		return fmt.Errorf("while opening %s: %w", itemFilePath, err)
-	}
-
-	defer itemFile.Close()
-
-	if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil {
-		return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
-	}
-
-	return nil
-}
-
-// downloadData downloads the data files for an item
-func downloadData(dataFolder string, force bool, reader io.Reader) error {
-	var err error
-
-	dec := yaml.NewDecoder(reader)
-
-	for {
-		data := &DataSet{}
-
-		err = dec.Decode(data)
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
-
-			return fmt.Errorf("while reading file: %w", err)
-		}
-
-		download := false
-
-		for _, dataS := range data.Data {
-			if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
-				download = true
-			}
-		}
-
-		if download || force {
-			err = GetData(data.Data, dataFolder)
-			if err != nil {
-				return fmt.Errorf("while getting data: %w", err)
-			}
-		}
-	}
-
-	return nil
-}

pkg/cwhub/download_test.go

@@ -1,60 +0,0 @@
-package cwhub
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"testing"
-
-	log "github.com/sirupsen/logrus"
-)
-
-func TestDownloadHubIdx(t *testing.T) {
-	back := RawFileURLTemplate
-	// bad url template
-	fmt.Println("Test 'bad URL'")
-
-	tmpIndex, err := os.CreateTemp("", "index.json")
-	if err != nil {
-		t.Fatalf("failed to create temp file : %s", err)
-	}
-
-	t.Cleanup(func() {
-		os.Remove(tmpIndex.Name())
-	})
-
-	RawFileURLTemplate = "x"
-
-	ret, err := DownloadHubIdx(tmpIndex.Name())
-	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
-		log.Errorf("unexpected error %s", err)
-	}
-
-	fmt.Printf("->%+v", ret)
-
-	// bad domain
-	fmt.Println("Test 'bad domain'")
-
-	RawFileURLTemplate = "https://baddomain/%s/%s"
-
-	ret, err = DownloadHubIdx(tmpIndex.Name())
-	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
-		log.Errorf("unexpected error %s", err)
-	}
-
-	fmt.Printf("->%+v", ret)
-
-	// bad target path
-	fmt.Println("Test 'bad target path'")
-
-	RawFileURLTemplate = back
-
-	ret, err = DownloadHubIdx("/does/not/exist/index.json")
-	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
-		log.Errorf("unexpected error %s", err)
-	}
-
-	RawFileURLTemplate = back
-
-	fmt.Printf("->%+v", ret)
-}

pkg/cwhub/install.go → pkg/cwhub/enable.go

@@ -1,5 +1,8 @@
 package cwhub
 
+// Enable/disable items already installed (no downloading here)
+// This file is not named install.go to avoid confusion with the functions in helpers.go
+
 import (
 	"fmt"
 	"os"
@@ -8,6 +11,81 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
+// Handles collections recursively
+func (h *Hub) EnableItem(target *Item) error {
+	var err error
+
+	parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
+
+	// create directories if needed
+	if target.Installed {
+		if target.Tainted {
+			return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
+		}
+
+		if target.Local {
+			return fmt.Errorf("%s is local, won't enable", target.Name)
+		}
+
+		// if it's a collection, check sub-items even if the collection file itself is up-to-date
+		if target.UpToDate && target.Type != COLLECTIONS {
+			log.Tracef("%s is installed and up-to-date, skip.", target.Name)
+			return nil
+		}
+	}
+
+	if _, err = os.Stat(parentDir); os.IsNotExist(err) {
+		log.Infof("%s doesn't exist, create", parentDir)
+
+		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
+			return fmt.Errorf("while creating directory: %w", err)
+		}
+	}
+
+	// install sub-items if it's a collection
+	if target.Type == COLLECTIONS {
+		for _, sub := range target.SubItems() {
+			val, ok := h.Items[sub.Type][sub.Name]
+			if !ok {
+				return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name)
+			}
+
+			err = h.EnableItem(&val)
+			if err != nil {
+				return fmt.Errorf("while installing %s: %w", sub.Name, err)
+			}
+		}
+	}
+
+	// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
+	if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) {
+		log.Infof("%s already exists.", parentDir+"/"+target.FileName)
+		return nil
+	}
+
+	// hub.ConfigDir + target.RemotePath
+	srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
+	if err != nil {
+		return fmt.Errorf("while getting source path: %w", err)
+	}
+
+	dstPath, err := filepath.Abs(parentDir + "/" + target.FileName)
+	if err != nil {
+		return fmt.Errorf("while getting destination path: %w", err)
+	}
+
+	if err = os.Symlink(srcPath, dstPath); err != nil {
+		return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
+	}
+
+	log.Infof("Enabled %s : %s", target.Type, target.Name)
+	target.Installed = true
+	h.Items[target.Type][target.Name] = *target
+
+	return nil
+}
+
 func (h *Hub) purgeItem(target Item) (Item, error) {
 	itempath := h.cfg.HubDir + "/" + target.RemotePath
 
@@ -49,31 +127,30 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
 
 	// for a COLLECTIONS, disable sub-items
 	if target.Type == COLLECTIONS {
-		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
-			ptrtype := ItemTypes[idx]
-			for _, p := range ptr {
-				if val, ok := h.Items[ptrtype][p]; ok {
-					// check if the item doesn't belong to another collection before removing it
-					toRemove := true
-
-					for _, collection := range val.BelongsToCollections {
-						if collection != target.Name {
-							toRemove = false
-							break
-						}
-					}
-
-					if toRemove {
-						err = h.DisableItem(&val, purge, force)
-						if err != nil {
-							return fmt.Errorf("while disabling %s: %w", p, err)
-						}
-					} else {
-						log.Infof("%s was not removed because it belongs to another collection", val.Name)
-					}
-				} else {
-					log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
+		for _, sub := range target.SubItems() {
+		       	val, ok := h.Items[sub.Type][sub.Name]
+			if !ok {
+		       		log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name)
+				continue
+		       	}
+
+			// check if the item doesn't belong to another collection before removing it
+			toRemove := true
+
+			for _, collection := range val.BelongsToCollections {
+				if collection != target.Name {
+					toRemove = false
+					break
+				}
+			}
+
+			if toRemove {
+				err = h.DisableItem(&val, purge, force)
+				if err != nil {
+					return fmt.Errorf("while disabling %s: %w", sub.Name, err)
 				}
+			} else {
+				log.Infof("%s was not removed because it belongs to another collection", val.Name)
 			}
 		}
 	}
@@ -132,81 +209,3 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
 
 	return nil
 }
-
-// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
-// Handles collections recursively
-func (h *Hub) EnableItem(target *Item) error {
-	var err error
-
-	parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
-
-	// create directories if needed
-	if target.Installed {
-		if target.Tainted {
-			return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
-		}
-
-		if target.Local {
-			return fmt.Errorf("%s is local, won't enable", target.Name)
-		}
-
-		// if it's a collection, check sub-items even if the collection file itself is up-to-date
-		if target.UpToDate && target.Type != COLLECTIONS {
-			log.Tracef("%s is installed and up-to-date, skip.", target.Name)
-			return nil
-		}
-	}
-
-	if _, err = os.Stat(parentDir); os.IsNotExist(err) {
-		log.Infof("%s doesn't exist, create", parentDir)
-
-		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
-			return fmt.Errorf("while creating directory: %w", err)
-		}
-	}
-
-	// install sub-items if it's a collection
-	if target.Type == COLLECTIONS {
-		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
-			ptrtype := ItemTypes[idx]
-			for _, p := range ptr {
-				val, ok := h.Items[ptrtype][p]
-				if !ok {
-					return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
-				}
-
-				err = h.EnableItem(&val)
-				if err != nil {
-					return fmt.Errorf("while installing %s: %w", p, err)
-				}
-			}
-		}
-	}
-
-	// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
-	if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) {
-		log.Infof("%s already exists.", parentDir+"/"+target.FileName)
-		return nil
-	}
-
-	// hub.ConfigDir + target.RemotePath
-	srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
-	if err != nil {
-		return fmt.Errorf("while getting source path: %w", err)
-	}
-
-	dstPath, err := filepath.Abs(parentDir + "/" + target.FileName)
-	if err != nil {
-		return fmt.Errorf("while getting destination path: %w", err)
-	}
-
-	if err = os.Symlink(srcPath, dstPath); err != nil {
-		return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
-	}
-
-	log.Infof("Enabled %s : %s", target.Type, target.Name)
-	target.Installed = true
-	h.Items[target.Type][target.Name] = *target
-
-	return nil
-}

pkg/cwhub/enable_test.go

@@ -0,0 +1,144 @@
+package cwhub
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testInstall(hub *Hub, t *testing.T, item Item) {
+	// Install the parser
+	err := hub.DownloadLatest(&item, false, false)
+	require.NoError(t, err, "failed to download %s", item.Name)
+
+	_, err = hub.LocalSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+
+	err = hub.EnableItem(&item)
+	require.NoError(t, err, "failed to enable %s", item.Name)
+
+	_, err = hub.LocalSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+}
+
+func testTaint(hub *Hub, t *testing.T, item Item) {
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+
+	f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
+	require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
+
+	defer f.Close()
+
+	_, err = f.WriteString("tainted")
+	require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
+
+	// Local sync and check status
+	_, err = hub.LocalSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
+}
+
+func testUpdate(hub *Hub, t *testing.T, item Item) {
+	assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
+
+	// Update it + check status
+	err := hub.DownloadLatest(&item, true, true)
+	require.NoError(t, err, "failed to update %s", item.Name)
+
+	// Local sync and check status
+	_, err = hub.LocalSync()
+	require.NoError(t, err, "failed to run localSync")
+
+	assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+}
+
+func testDisable(hub *Hub, t *testing.T, item Item) {
+	assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+
+	// Remove
+	err := hub.DisableItem(&item, false, false)
+	require.NoError(t, err, "failed to disable %s", item.Name)
+
+	// Local sync and check status
+	warns, err := hub.LocalSync()
+	require.NoError(t, err, "failed to run localSync")
+	require.Empty(t, warns, "unexpected warnings : %+v", warns)
+
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
+
+	// Purge
+	err = hub.DisableItem(&item, true, false)
+	require.NoError(t, err, "failed to purge %s", item.Name)
+
+	// Local sync and check status
+	warns, err = hub.LocalSync()
+	require.NoError(t, err, "failed to run localSync")
+	require.Empty(t, warns, "unexpected warnings : %+v", warns)
+
+	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
+}
+
+func TestInstallParser(t *testing.T) {
+	/*
+	 - install a random parser
+	 - check its status
+	 - taint it
+	 - check its status
+	 - force update it
+	 - check its status
+	 - remove it
+	*/
+	hub := envSetup(t)
+
+	// map iteration is random by itself
+	for _, it := range hub.Items[PARSERS] {
+		testInstall(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testTaint(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testUpdate(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testDisable(hub, t, it)
+
+		break
+	}
+}
+
+func TestInstallCollection(t *testing.T) {
+	/*
+	 - install a random parser
+	 - check its status
+	 - taint it
+	 - check its status
+	 - force update it
+	 - check its status
+	 - remove it
+	*/
+	hub := envSetup(t)
+
+	// map iteration is random by itself
+	for _, it := range hub.Items[COLLECTIONS] {
+		testInstall(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testTaint(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testUpdate(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testDisable(hub, t, it)
+
+		break
+	}
+}

pkg/cwhub/helpers.go

@@ -1,65 +1,23 @@
 package cwhub
 
+// Install, upgrade and remove items from the hub to the local configuration
+
+// XXX: this file could use a better name
+
 import (
+	"bytes"
+	"crypto/sha256"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/enescakir/emoji"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/mod/semver"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 )
 
-// chooseHubBranch returns the branch name to use for the hub
-// It can be "master" or branch corresponding to the current crowdsec version
-func chooseHubBranch() string {
-	latest, err := cwversion.Latest()
-	if err != nil {
-		log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
-		return "master"
-	}
-
-	csVersion := cwversion.VersionStrip()
-	if csVersion == latest {
-		log.Debugf("current version is equal to latest (%s)", csVersion)
-		return "master"
-	}
-
-	// if current version is greater than the latest we are in pre-release
-	if semver.Compare(csVersion, latest) == 1 {
-		log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
-		return "master"
-	}
-
-	if csVersion == "" {
-		log.Warning("Crowdsec version is not set, using master branch for the hub")
-		return "master"
-	}
-
-	log.Warnf("Crowdsec is not the latest version. "+
-		"Current version is '%s' and the latest stable version is '%s'. Please update it!",
-		csVersion, latest)
-
-	log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
-		"added to Crowdsec Hub after CrowdSec %s", latest)
-
-	return csVersion
-}
-
-// SetHubBranch sets the package variable that points to the hub branch.
-func SetHubBranch() {
-	// a branch is already set, or specified from the flags
-	if HubBranch != "" {
-		return
-	}
-
-	// use the branch corresponding to the crowdsec version
-	HubBranch = chooseHubBranch()
-
-	log.Debugf("Using branch '%s' for the hub", HubBranch)
-}
-
 // InstallItem installs an item from the hub
 func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error {
 	item := h.GetItem(itemType, name)
@@ -80,7 +38,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
 		return fmt.Errorf("while downloading %s: %w", item.Name, err)
 	}
 
-	if err = h.AddItem(itemType, *item); err != nil {
+	if err = h.AddItem(*item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
@@ -94,7 +52,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
 		return fmt.Errorf("while enabling %s: %w", item.Name, err)
 	}
 
-	if err := h.AddItem(itemType, *item); err != nil {
+	if err := h.AddItem(*item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
@@ -117,7 +75,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for
 			return fmt.Errorf("unable to disable %s: %w", item.Name, err)
 		}
 
-		if err = h.AddItem(itemType, *item); err != nil {
+		if err = h.AddItem(*item); err != nil {
 			return fmt.Errorf("unable to add %s: %w", item.Name, err)
 		}
 
@@ -141,7 +99,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for
 			return fmt.Errorf("unable to disable %s: %w", v.Name, err)
 		}
 
-		if err := h.AddItem(itemType, v); err != nil {
+		if err := h.AddItem(v); err != nil {
 			return fmt.Errorf("unable to add %s: %w", v.Name, err)
 		}
 		disabled++
@@ -204,7 +162,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error {
 			updated++
 		}
 
-		if err := h.AddItem(itemType, v); err != nil {
+		if err := h.AddItem(v); err != nil {
 			return fmt.Errorf("unable to add %s: %w", v.Name, err)
 		}
 	}
@@ -225,3 +183,192 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error {
 
 	return nil
 }
+
+// DownloadLatest will download the latest version of Item to the tdir directory
+func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error {
+	var err error
+
+	log.Debugf("Downloading %s %s", target.Type, target.Name)
+
+	if target.Type != COLLECTIONS {
+		if !target.Installed && updateOnly && target.Downloaded {
+			log.Debugf("skipping upgrade of %s : not installed", target.Name)
+			return nil
+		}
+
+		return h.DownloadItem(target, overwrite)
+	}
+
+	// collection
+	for _, sub := range target.SubItems() {
+		val, ok := h.Items[sub.Type][sub.Name]
+		if !ok {
+			return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name)
+		}
+
+		if !val.Installed && updateOnly && val.Downloaded {
+			log.Debugf("skipping upgrade of %s : not installed", target.Name)
+			continue
+		}
+
+		log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly)
+		//recurse as it's a collection
+		if sub.Type == COLLECTIONS {
+			log.Tracef("collection, recurse")
+
+			err = h.DownloadLatest(&val, overwrite, updateOnly)
+			if err != nil {
+				return fmt.Errorf("while downloading %s: %w", val.Name, err)
+			}
+		}
+
+		downloaded := val.Downloaded
+
+		err = h.DownloadItem(&val, overwrite)
+		if err != nil {
+			return fmt.Errorf("while downloading %s: %w", val.Name, err)
+		}
+
+		// We need to enable an item when it has been added to a collection since latest release of the collection.
+		// We check if val.Downloaded is false because maybe the item has been disabled by the user.
+		if !val.Installed && !downloaded {
+			if err = h.EnableItem(&val); err != nil {
+				return fmt.Errorf("enabling '%s': %w", val.Name, err)
+			}
+		}
+
+		h.Items[sub.Type][sub.Name] = val
+	}
+
+	err = h.DownloadItem(target, overwrite)
+	if err != nil {
+		return fmt.Errorf("failed to download item: %w", err)
+	}
+
+	return nil
+}
+
+func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
+	tdir := h.cfg.HubDir
+
+	// if user didn't --force, don't overwrite local, tainted, up-to-date files
+	if !overwrite {
+		if target.Tainted {
+			log.Debugf("%s : tainted, not updated", target.Name)
+			return nil
+		}
+
+		if target.UpToDate {
+			//  We still have to check if data files are present
+			log.Debugf("%s : up-to-date, not updated", target.Name)
+		}
+	}
+
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
+	if err != nil {
+		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
+	}
+
+	hash := sha256.New()
+	if _, err = hash.Write(body); err != nil {
+		return fmt.Errorf("while hashing %s: %w", target.Name, err)
+	}
+
+	meow := fmt.Sprintf("%x", hash.Sum(nil))
+	if meow != target.Versions[target.Version].Digest {
+		log.Errorf("Downloaded version doesn't match index, please 'hub update'")
+		log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
+
+		return fmt.Errorf("invalid download hash for %s", target.Name)
+	}
+
+	//all good, install
+	//check if parent dir exists
+	tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
+	parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
+
+	// ensure that target file is within target dir
+	finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
+	if err != nil {
+		return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
+	}
+
+	if !strings.HasPrefix(finalPath, tdir) {
+		return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
+	}
+
+	// check dir
+	if _, err = os.Stat(parentDir); os.IsNotExist(err) {
+		log.Debugf("%s doesn't exist, create", parentDir)
+
+		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
+			return fmt.Errorf("while creating parent directories: %w", err)
+		}
+	}
+
+	// check actual file
+	if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
+		log.Warningf("%s : overwrite", target.Name)
+		log.Debugf("target: %s/%s", tdir, target.RemotePath)
+	} else {
+		log.Infof("%s : OK", target.Name)
+	}
+
+	f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		return fmt.Errorf("while opening file: %w", err)
+	}
+
+	defer f.Close()
+
+	_, err = f.Write(body)
+	if err != nil {
+		return fmt.Errorf("while writing file: %w", err)
+	}
+
+	target.Downloaded = true
+	target.Tainted = false
+	target.UpToDate = true
+
+	if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
+		return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
+	}
+
+	h.Items[target.Type][target.Name] = *target
+
+	return nil
+}
+
+// DownloadDataIfNeeded downloads the data files for an item
+func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
+	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName)
+
+	itemFile, err := os.Open(itemFilePath)
+	if err != nil {
+		return fmt.Errorf("while opening %s: %w", itemFilePath, err)
+	}
+
+	defer itemFile.Close()
+
+	if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil {
+		return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
+	}
+
+	return nil
+}

pkg/cwhub/helpers_test.go

@@ -14,8 +14,6 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
 	hub := envSetup(t)
 
 	// fresh install of collection
-	hub = getHubOrFail(t, hub.cfg)
-
 	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
 	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
 
@@ -59,8 +57,6 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	hub := envSetup(t)
 
 	// fresh install of collection
-	hub = getHubOrFail(t, hub.cfg)
-
 	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
 	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
 	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
@@ -99,6 +95,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub {
 	hub, err := InitHub(hubCfg)
 	require.NoError(t, err, "failed to load hub index")
+
 	return hub
 }
 
@@ -109,8 +106,6 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
 	hub := envSetup(t)
 
 	// fresh install of collection
-	hub = getHubOrFail(t, hub.cfg)
-
 	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
 	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
 	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)

pkg/cwhub/hub.go

@@ -1,8 +1,13 @@
 package cwhub
 
 import (
+	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -10,22 +15,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
-const (
-	HubIndexFile = ".index.json"
-
-	// managed item types
-	COLLECTIONS   = "collections"
-	PARSERS       = "parsers"
-	POSTOVERFLOWS = "postoverflows"
-	SCENARIOS     = "scenarios"
-)
-
-var (
-	// XXX: The order is important, as it is used to range over sub-items in collections
-	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
-)
-
-type HubItems map[string]map[string]Item
+const HubIndexFile = ".index.json"
 
 // Hub represents the runtime status of the hub (parsed items, etc.)
 type Hub struct {
@@ -35,7 +25,10 @@ type Hub struct {
 	skippedTainted int
 }
 
-var theHub *Hub
+var (
+	theHub           *Hub
+	ErrIndexNotFound = fmt.Errorf("index not found")
+)
 
 // GetHub returns the hub singleton
 // it returns an error if it's not initialized to avoid nil dereference
@@ -47,32 +40,129 @@ func GetHub() (*Hub, error) {
 	return theHub, nil
 }
 
-// displaySummary prints a total count of the hub items
-func (h Hub) displaySummary() {
-	msg := "Loaded: "
-	for itemType := range h.Items {
-		msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
+// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use
+func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("no configuration found for hub")
 	}
-	log.Info(strings.Trim(msg, ", "))
 
-	if h.skippedLocal > 0 || h.skippedTainted > 0 {
-		log.Infof("unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted)
+	log.Debugf("loading hub idx %s", cfg.HubIndexFile)
+
+	bidx, err := os.ReadFile(cfg.HubIndexFile)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read index file: %w", err)
+	}
+
+	ret, err := ParseIndex(bidx)
+	if err != nil {
+		if !errors.Is(err, ErrMissingReference) {
+			return nil, fmt.Errorf("unable to load existing index: %w", err)
+		}
+
+		// XXX: why the error check if we bail out anyway?
+		return nil, err
+	}
+
+	theHub = &Hub{
+		Items: ret,
+		cfg:   cfg,
 	}
+
+	_, err = theHub.LocalSync()
+	if err != nil {
+		return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
+	}
+
+	return theHub, nil
 }
 
-// DisplaySummary prints a total count of the hub items.
-// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton
-// XXX: to be removed later
-func DisplaySummary() error {
-	hub, err := GetHub()
+// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk
+// It is used to inizialize the hub when there is no index file yet
+func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("no configuration found for hub")
+	}
+
+	bidx, err := DownloadIndex(cfg.HubIndexFile)
 	if err != nil {
-		return err
+		return nil, fmt.Errorf("failed to download index: %w", err)
 	}
-	hub.displaySummary()
-	return nil
+
+	ret, err := ParseIndex(bidx)
+	if err != nil {
+		if !errors.Is(err, ErrMissingReference) {
+			return nil, fmt.Errorf("failed to read index: %w", err)
+		}
+	}
+
+	theHub = &Hub{
+		Items: ret,
+		cfg:   cfg,
+	}
+
+	if _, err := theHub.LocalSync(); err != nil {
+		return nil, fmt.Errorf("failed to sync: %w", err)
+	}
+
+	return theHub, nil
 }
 
-// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections
+// DownloadIndex downloads the latest version of the index and returns the content
+func DownloadIndex(indexPath string) ([]byte, error) {
+	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
+
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build request for hub index: %w", err)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed http request for hub index: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusNotFound {
+			return nil, ErrIndexNotFound
+		}
+
+		return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
+	}
+
+	oldContent, err := os.ReadFile(indexPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			log.Warningf("failed to read hub index: %s", err)
+		}
+	} else if bytes.Equal(body, oldContent) {
+		log.Info("hub index is up to date")
+		// write it anyway, can't hurt
+	}
+
+	file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+
+	if err != nil {
+		return nil, fmt.Errorf("while opening hub index file: %w", err)
+	}
+	defer file.Close()
+
+	wsize, err := file.Write(body)
+	if err != nil {
+		return nil, fmt.Errorf("while writing hub index file: %w", err)
+	}
+
+	log.Infof("Wrote new %d bytes index to %s", wsize, indexPath)
+
+	return body, nil
+}
+
+// ParseIndex takes the content of an index file and returns the map of associated parsers/scenarios/collections
 func ParseIndex(buff []byte) (HubItems, error) {
 	var (
 		RawIndex     HubItems
@@ -102,13 +192,10 @@ func ParseIndex(buff []byte) (HubItems, error) {
 
 			// if it's a collection, check its sub-items are present
 			// XXX should be done later
-			for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
-				ptrtype := ItemTypes[idx]
-				for _, p := range ptr {
-					if _, ok := RawIndex[ptrtype][p]; !ok {
-						log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
-						missingItems = append(missingItems, p)
-					}
+			for _, sub := range item.SubItems() {
+				if _, ok := RawIndex[sub.Type][sub.Name]; !ok {
+					log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, item.Name)
+					missingItems = append(missingItems, sub.Name)
 				}
 			}
 		}
@@ -120,3 +207,34 @@ func ParseIndex(buff []byte) (HubItems, error) {
 
 	return RawIndex, nil
 }
+
+// ItemStats returns total counts of the hub items
+func (h Hub) ItemStats() []string {
+	loaded := ""
+	for _, itemType := range ItemTypes {
+		// ensure the order is always the same
+		if h.Items[itemType] == nil {
+			continue
+		}
+		if len(h.Items[itemType]) == 0 {
+			continue
+		}
+		loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
+	}
+
+	loaded = strings.Trim(loaded, ", ")
+	if loaded == "" {
+		// empty hub
+		loaded = "0 items"
+	}
+
+	ret := []string{
+		fmt.Sprintf("Loaded: %s", loaded),
+	}
+
+	if h.skippedLocal > 0 || h.skippedTainted > 0 {
+		ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted))
+	}
+
+	return ret
+}

pkg/cwhub/hub_test.go

@@ -0,0 +1,63 @@
+package cwhub
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
+)
+
+func TestInitHubUpdate(t *testing.T) {
+	hub := envSetup(t)
+
+	_, err := InitHubUpdate(hub.cfg)
+	require.NoError(t, err)
+
+	_, err = GetHub()
+	require.NoError(t, err)
+}
+
+func TestDownloadIndex(t *testing.T) {
+	back := RawFileURLTemplate
+	// bad url template
+	fmt.Println("Test 'bad URL'")
+
+	tmpIndex, err := os.CreateTemp("", "index.json")
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		os.Remove(tmpIndex.Name())
+	})
+
+	RawFileURLTemplate = "x"
+
+	ret, err := DownloadIndex(tmpIndex.Name())
+	cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ")
+
+	fmt.Printf("->%+v", ret)
+
+	// bad domain
+	fmt.Println("Test 'bad domain'")
+
+	RawFileURLTemplate = "https://baddomain/%s/%s"
+
+	ret, err = DownloadIndex(tmpIndex.Name())
+	cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
+
+	fmt.Printf("->%+v", ret)
+
+	// bad target path
+	fmt.Println("Test 'bad target path'")
+
+	RawFileURLTemplate = back
+
+	ret, err = DownloadIndex("/does/not/exist/index.json")
+	cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:")
+
+	RawFileURLTemplate = back
+
+	fmt.Printf("->%+v", ret)
+}

pkg/cwhub/items.go

@@ -0,0 +1,232 @@
+package cwhub
+
+import (
+	"fmt"
+
+	"github.com/enescakir/emoji"
+	"golang.org/x/mod/semver"
+)
+
+const (
+	// managed item types
+	COLLECTIONS   = "collections"
+	PARSERS       = "parsers"
+	POSTOVERFLOWS = "postoverflows"
+	SCENARIOS     = "scenarios"
+)
+
+// XXX: The order is important, as it is used to range over sub-items in collections
+var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
+
+type HubItems map[string]map[string]Item
+
+// ItemVersion is used to detect the version of a given item
+// by comparing the hash of each version to the local file.
+// If the item does not match any known version, it is considered tainted.
+type ItemVersion struct {
+	Digest     string `json:"digest,omitempty"`     // meow
+	Deprecated bool   `json:"deprecated,omitempty"` // XXX: do we keep this?
+}
+
+// Item represents an object managed in the hub. It can be a parser, scenario, collection..
+type Item struct {
+	// descriptive info
+	Type                 string   `json:"type,omitempty"                   yaml:"type,omitempty"`                   // can be any of the ItemTypes
+	Stage                string   `json:"stage,omitempty"                  yaml:"stage,omitempty"`                  // Stage for parser|postoverflow: s00-raw/s01-...
+	Name                 string   `json:"name,omitempty"`                                                           // as seen in .index.json, usually "author/name"
+	FileName             string   `json:"file_name,omitempty"`                                                      // the filename, ie. apache2-logs.yaml
+	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .index.json
+	Author               string   `json:"author,omitempty"`                                                         // as seen in .index.json
+	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`             // as seen in .index.json
+	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
+
+	// remote (hub) info
+	RemotePath string                 `json:"path,omitempty"      yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
+	Version    string                 `json:"version,omitempty"`                                // the last version
+	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                     // the list of existing versions
+
+	// local (deployed) info
+	LocalPath    string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
+	LocalVersion string `json:"local_version,omitempty"`
+	LocalHash    string `json:"local_hash,omitempty"` // the local meow
+	Installed    bool   `json:"installed,omitempty"`
+	Downloaded   bool   `json:"downloaded,omitempty"`
+	UpToDate     bool   `json:"up_to_date,omitempty"`
+	Tainted      bool   `json:"tainted,omitempty"` // has it been locally modified?
+	Local        bool   `json:"local,omitempty"`   // if it's a non versioned control one
+
+	// if it's a collection, it can have sub items
+	Parsers       []string `json:"parsers,omitempty"       yaml:"parsers,omitempty"`
+	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
+	Scenarios     []string `json:"scenarios,omitempty"     yaml:"scenarios,omitempty"`
+	Collections   []string `json:"collections,omitempty"   yaml:"collections,omitempty"`
+}
+
+
+type SubItem struct {
+	Type string
+	Name string
+}
+
+func (i *Item) SubItems() []SubItem {
+	sub := make([]SubItem,
+		len(i.Parsers) +
+		len(i.PostOverflows) +
+		len(i.Scenarios) +
+		len(i.Collections))
+	n := 0
+	for _, name := range i.Parsers {
+		sub[n] = SubItem{Type: PARSERS, Name: name}
+		n++
+	}
+	for _, name := range i.PostOverflows {
+		sub[n] = SubItem{Type: POSTOVERFLOWS, Name: name}
+		n++
+	}
+	for _, name := range i.Scenarios {
+		sub[n] = SubItem{Type: SCENARIOS, Name: name}
+		n++
+	}
+	for _, name := range i.Collections {
+		sub[n] = SubItem{Type: COLLECTIONS, Name: name}
+		n++
+	}
+	return sub
+}
+
+// Status returns the status of the item as a string and an emoji
+// ie. "enabled,update-available" and emoji.Warning
+func (i *Item) Status() (string, emoji.Emoji) {
+	status := "disabled"
+	ok := false
+
+	if i.Installed {
+		ok = true
+		status = "enabled"
+	}
+
+	managed := true
+	if i.Local {
+		managed = false
+		status += ",local"
+	}
+
+	warning := false
+	if i.Tainted {
+		warning = true
+		status += ",tainted"
+	} else if !i.UpToDate && !i.Local {
+		warning = true
+		status += ",update-available"
+	}
+
+	emo := emoji.QuestionMark
+
+	switch {
+	case !managed:
+		emo = emoji.House
+	case !i.Installed:
+		emo = emoji.Prohibited
+	case warning:
+		emo = emoji.Warning
+	case ok:
+		emo = emoji.CheckMark
+	}
+
+	return status, emo
+}
+
+// versionStatus: semver requires 'v' prefix
+func (i *Item) versionStatus() int {
+	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
+}
+
+// validPath returns true if the (relative) path is allowed for the item
+// dirNmae: the directory name (ie. crowdsecurity)
+// fileName: the filename (ie. apache2-logs.yaml)
+func (i *Item) validPath(dirName, fileName string) bool {
+	return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
+}
+
+// GetItemMap returns the map of items for a given type
+func (h *Hub) GetItemMap(itemType string) map[string]Item {
+	m, ok := h.Items[itemType]
+	if !ok {
+		return nil
+	}
+
+	return m
+}
+
+// GetItem returns the item from hub based on its type and full name (author/name)
+func (h *Hub) GetItem(itemType string, itemName string) *Item {
+	m, ok := h.GetItemMap(itemType)[itemName]
+	if !ok {
+		return nil
+	}
+
+	return &m
+}
+
+// GetItemNames returns the list of item (full) names for a given type
+// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
+// The names can be used to retrieve the item with GetItem()
+func (h *Hub) GetItemNames(itemType string) []string {
+	m := h.GetItemMap(itemType)
+	if m == nil {
+		return nil
+	}
+
+	names := make([]string, 0, len(m))
+	for k := range m {
+		names = append(names, k)
+	}
+
+	return names
+}
+
+// AddItem adds an item to the hub index
+func (h *Hub) AddItem(item Item) error {
+	for _, t := range ItemTypes {
+		if t == item.Type {
+			h.Items[t][item.Name] = item
+			return nil
+		}
+	}
+
+	return fmt.Errorf("ItemType %s is unknown", item.Type)
+}
+
+// GetInstalledItems returns the list of installed items
+func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) {
+	items, ok := h.Items[itemType]
+	if !ok {
+		return nil, fmt.Errorf("no %s in hubIdx", itemType)
+	}
+
+	retItems := make([]Item, 0)
+
+	for _, item := range items {
+		if item.Installed {
+			retItems = append(retItems, item)
+		}
+	}
+
+	return retItems, nil
+}
+
+// GetInstalledItemsAsString returns the names of the installed items
+func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
+	items, err := h.GetInstalledItems(itemType)
+	if err != nil {
+		return nil, err
+	}
+
+	retStr := make([]string, len(items))
+
+	for i, it := range items {
+		retStr[i] = it.Name
+	}
+
+	return retStr, nil
+}

pkg/cwhub/items_test.go

@@ -0,0 +1,75 @@
+package cwhub
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/go-cs-lib/cstest"
+)
+
+func TestItemStatus(t *testing.T) {
+	hub := envSetup(t)
+
+	// get existing map
+	x := hub.GetItemMap(COLLECTIONS)
+	require.NotEmpty(t, x)
+
+	// Get item : good and bad
+	for k := range x {
+		item := hub.GetItem(COLLECTIONS, k)
+		require.NotNil(t, item)
+
+		item.Installed = true
+		item.UpToDate = false
+		item.Local = false
+		item.Tainted = false
+
+		txt, _ := item.Status()
+		require.Equal(t, "enabled,update-available", txt)
+
+		item.Installed = false
+		item.UpToDate = false
+		item.Local = true
+		item.Tainted = false
+
+		txt, _ = item.Status()
+		require.Equal(t, "disabled,local", txt)
+	}
+
+	stats := hub.ItemStats()
+	require.Equal(t, []string{"Loaded: 2 parsers, 1 scenarios, 3 collections"}, stats)
+}
+
+func TestGetters(t *testing.T) {
+	hub := envSetup(t)
+
+	// get non existing map
+	empty := hub.GetItemMap("ratata")
+	require.Nil(t, empty)
+
+	// get existing map
+	x := hub.GetItemMap(COLLECTIONS)
+	require.NotEmpty(t, x)
+
+	// Get item : good and bad
+	for k := range x {
+		empty := hub.GetItem(COLLECTIONS, k+"nope")
+		require.Nil(t, empty)
+
+		item := hub.GetItem(COLLECTIONS, k)
+		require.NotNil(t, item)
+
+		// Add item and get it
+		item.Name += "nope"
+		err := hub.AddItem(*item)
+		require.NoError(t, err)
+
+		newitem := hub.GetItem(COLLECTIONS, item.Name)
+		require.NotNil(t, newitem)
+
+		item.Type = "ratata"
+		err = hub.AddItem(*item)
+		cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
+	}
+}

pkg/cwhub/leakybucket.go

@@ -0,0 +1,58 @@
+package cwhub
+
+// Resolve a symlink to find the hub item it points to.
+// This file is used only by pkg/leakybucket
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
+func itemKey(itemPath string) (string, error) {
+	f, err := os.Lstat(itemPath)
+	if err != nil {
+		return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
+	}
+
+	if f.Mode()&os.ModeSymlink == 0 {
+		// it's not a symlink, so the filename itsef should be the key
+		return filepath.Base(itemPath), nil
+	}
+
+	// resolve the symlink to hub file
+	pathInHub, err := os.Readlink(itemPath)
+	if err != nil {
+		return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
+	}
+
+	author := filepath.Base(filepath.Dir(pathInHub))
+
+	fname := filepath.Base(pathInHub)
+	fname = strings.TrimSuffix(fname, ".yaml")
+	fname = strings.TrimSuffix(fname, ".yml")
+
+	return fmt.Sprintf("%s/%s", author, fname), nil
+}
+
+// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
+func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
+	itemKey, err := itemKey(itemPath)
+	if err != nil {
+		return nil, err
+	}
+
+	m := h.GetItemMap(itemType)
+	if m == nil {
+		return nil, fmt.Errorf("item type %s doesn't exist", itemType)
+	}
+
+	v, ok := m[itemKey]
+	if !ok {
+		return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
+	}
+
+	return &v, nil
+}

pkg/cwhub/loader.go → pkg/cwhub/sync.go

@@ -2,7 +2,6 @@ package cwhub
 
 import (
 	"crypto/sha256"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -11,8 +10,6 @@ import (
 	"strings"
 
 	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
 func isYAMLFileName(path string) bool {
@@ -109,7 +106,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
 
 	log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
 	// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
-
+	
 	if ret.stage == SCENARIOS {
 		ret.ftype = SCENARIOS
 		ret.stage = ""
@@ -325,66 +322,63 @@ func (h *Hub) CollectDepsCheck(v *Item) error {
 	// if it's a collection, ensure all the items are installed, or tag it as tainted
 	log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
 
-	for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
-		sliceType := ItemTypes[idx]
-		for _, subName := range itemSlice {
-			subItem, ok := h.Items[sliceType][subName]
-			if !ok {
-				return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name)
-			}
-
-			log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
+	for _, sub := range v.SubItems() {
+		subItem, ok := h.Items[sub.Type][sub.Name]
+		if !ok {
+			return fmt.Errorf("referred %s %s in collection %s doesn't exist", sub.Type, sub.Name, v.Name)
+		}
 
-			if !v.Installed {
-				continue
-			}
+		log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
 
-			if subItem.Type == COLLECTIONS {
-				log.Tracef("collec, recurse.")
+		if !v.Installed {
+			continue
+		}
 
-				if err := h.CollectDepsCheck(&subItem); err != nil {
-					if subItem.Tainted {
-						v.Tainted = true
-					}
+		if subItem.Type == COLLECTIONS {
+			log.Tracef("collec, recurse.")
 
-					return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
+			if err := h.CollectDepsCheck(&subItem); err != nil {
+				if subItem.Tainted {
+					v.Tainted = true
 				}
 
-				h.Items[sliceType][subName] = subItem
+				return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
 			}
 
-			// propagate the state of sub-items to set
-			if subItem.Tainted {
-				v.Tainted = true
-				return fmt.Errorf("tainted %s %s, tainted", sliceType, subName)
-			}
+			h.Items[sub.Type][sub.Name] = subItem
+		}
 
-			if !subItem.Installed && v.Installed {
-				v.Tainted = true
-				return fmt.Errorf("missing %s %s, tainted", sliceType, subName)
-			}
+		// propagate the state of sub-items to set
+		if subItem.Tainted {
+			v.Tainted = true
+			return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name)
+		}
 
-			if !subItem.UpToDate {
-				v.UpToDate = false
-				return fmt.Errorf("outdated %s %s", sliceType, subName)
-			}
+		if !subItem.Installed && v.Installed {
+			v.Tainted = true
+			return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name)
+		}
 
-			skip := false
+		if !subItem.UpToDate {
+			v.UpToDate = false
+			return fmt.Errorf("outdated %s %s", sub.Type, sub.Name)
+		}
 
-			for idx := range subItem.BelongsToCollections {
-				if subItem.BelongsToCollections[idx] == v.Name {
-					skip = true
-				}
-			}
+		skip := false
 
-			if !skip {
-				subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
+		for idx := range subItem.BelongsToCollections {
+			if subItem.BelongsToCollections[idx] == v.Name {
+				skip = true
 			}
+		}
 
-			h.Items[sliceType][subName] = subItem
-
-			log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
+		if !skip {
+			subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
 		}
+
+		h.Items[sub.Type][sub.Name] = subItem
+
+		log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate)
 	}
 
 	return nil
@@ -447,39 +441,3 @@ func (h *Hub) LocalSync() ([]string, error) {
 
 	return warnings, nil
 }
-
-// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use
-func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
-	if cfg == nil {
-		return nil, fmt.Errorf("no configuration found for hub")
-	}
-
-	log.Debugf("loading hub idx %s", cfg.HubIndexFile)
-
-	bidx, err := os.ReadFile(cfg.HubIndexFile)
-	if err != nil {
-		return nil, fmt.Errorf("unable to read index file: %w", err)
-	}
-
-	ret, err := ParseIndex(bidx)
-	if err != nil {
-		if !errors.Is(err, ErrMissingReference) {
-			return nil, fmt.Errorf("unable to load existing index: %w", err)
-		}
-
-		// XXX: why the error check if we bail out anyway?
-		return nil, err
-	}
-
-	theHub = &Hub{
-		Items: ret,
-		cfg:   cfg,
-	}
-
-	_, err = theHub.LocalSync()
-	if err != nil {
-		return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
-	}
-
-	return theHub, nil
-}

pkg/hubtest/utils.go

@@ -22,7 +22,7 @@ func Copy(src string, dst string) error {
 		return err
 	}
 
-	err = os.WriteFile(dst, content, 0644)
+	err = os.WriteFile(dst, content, 0o644)
 	if err != nil {
 		return err
 	}

test/bats/20_hub_collections.bats

@@ -258,6 +258,14 @@ teardown() {
     assert_output "0"
 }
 
+@test "cscli parsers remove [parser]... --force" {
+    # remove a parser that belongs to a collection
+    rune -0 cscli collections install crowdsecurity/linux
+    rune -0 cscli collections remove crowdsecurity/sshd
+    assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/linux]"
+    assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection"
+}
+
 @test "cscli collections upgrade [collection]..." {
     rune -1 cscli collections upgrade
     assert_stderr --partial "specify at least one collection to upgrade or '--all'"

test/bats/20_hub_collections_dep.bats

@@ -49,8 +49,8 @@ teardown() {
     rune -0 cscli collections install crowdsecurity/smb
     # XXX: should this be an error?
     rune -0 cscli collections remove crowdsecurity/sshd
-    assert_stderr --partial "crowdsecurity/sshd belongs to other collections: [crowdsecurity/smb]"
-    assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this sub collection"
+    assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/smb]"
+    assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection"
     rune -0 cscli collections list -o json
     rune -0 jq -c '[.collections[].name]' <(output)
     assert_json '["crowdsecurity/smb","crowdsecurity/sshd"]'

test/bats/20_hub_parsers.bats

@@ -268,6 +268,14 @@ teardown() {
     assert_output "0"
 }
 
+@test "cscli parsers remove [parser]... --force" {
+    # remove a parser that belongs to a collection
+    rune -0 cscli collections install crowdsecurity/linux
+    rune -0 cscli parsers remove crowdsecurity/sshd-logs
+    assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]"
+    assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser"
+}
+
 @test "cscli parsers upgrade [parser]..." {
     rune -1 cscli parsers upgrade
     assert_stderr --partial "specify at least one parser to upgrade or '--all'"

test/bats/20_hub_postoverflows.bats

@@ -260,6 +260,14 @@ teardown() {
     assert_output "0"
 }
 
+@test "cscli postoverflows remove [parser]... --force" {
+    # remove a parser that belongs to a collection
+    rune -0 cscli collections install crowdsecurity/auditd
+    rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process
+    assert_stderr --partial "crowdsecurity/auditd-whitelisted-process belongs to collections: [crowdsecurity/auditd]"
+    assert_stderr --partial "Run 'sudo cscli postoverflows remove crowdsecurity/auditd-whitelisted-process --force' if you want to force remove this postoverflow"
+}
+
 @test "cscli postoverflows upgrade [postoverflow]..." {
     rune -1 cscli postoverflows upgrade
     assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'"

test/bats/20_hub_scenarios.bats

@@ -260,6 +260,14 @@ teardown() {
     assert_output "0"
 }
 
+@test "cscli scenarios remove [scenario]... --force" {
+    # remove a scenario that belongs to a collection
+    rune -0 cscli collections install crowdsecurity/sshd
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf
+    assert_stderr --partial "crowdsecurity/ssh-bf belongs to collections: [crowdsecurity/sshd]"
+    assert_stderr --partial "Run 'sudo cscli scenarios remove crowdsecurity/ssh-bf --force' if you want to force remove this scenario"
+}
+
 @test "cscli scenarios upgrade [scenario]..." {
     rune -1 cscli scenarios upgrade
     assert_stderr --partial "specify at least one scenario to upgrade or '--all'"