Refact pkg/csconfig, pkg/cwhub (#2555)

* csconfig: drop redundant hub information on *Cfg structs
* rename validItemFileName() -> item.validPath()
* Methods on hub object
* updated tests to reduce need of csconfig.Config or global state

cmd/crowdsec-cli/capi.go

@@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command {
 				return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig)
+			if err != nil {
 				return err
 			}
 
-			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+			scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 			if err != nil {
 				return fmt.Errorf("failed to get scenarios: %w", err)
 			}

cmd/crowdsec-cli/collections.go

@@ -25,7 +25,7 @@ cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
 		Aliases:           []string{"collection"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
+			if _, err := require.Hub(csConfig); err != nil {
 				return err
 			}
 
@@ -66,8 +66,13 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	for _, name := range args {
-		t := cwhub.GetItem(cwhub.COLLECTIONS, name)
+		t := hub.GetItem(cwhub.COLLECTIONS, name)
 		if t == nil {
 			nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
 			Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
@@ -75,7 +80,7 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error {
 			continue
 		}
 
-		if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, force, downloadOnly); err != nil {
+		if err := hub.InstallItem(name, cwhub.COLLECTIONS, force, downloadOnly); err != nil {
 			if !ignoreError {
 				return fmt.Errorf("error while installing '%s': %w", name, err)
 			}
@@ -126,8 +131,13 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, force)
+		err := hub.RemoveMany(cwhub.COLLECTIONS, "", all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -141,7 +151,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error {
 
 	for _, name := range args {
 		if !force {
-			item := cwhub.GetItem(cwhub.COLLECTIONS, name)
+			item := hub.GetItem(cwhub.COLLECTIONS, name)
 			if item == nil {
 				// XXX: this should be in GetItem?
 				return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS)
@@ -153,7 +163,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error {
 			}
 		}
 
-		err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, force)
+		err := hub.RemoveMany(cwhub.COLLECTIONS, name, all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -197,8 +207,13 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil {
 			return err
 		}
 		return nil
@@ -209,7 +224,7 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.COLLECTIONS, name, force); err != nil {
 			return err
 		}
 	}

cmd/crowdsec-cli/config_backup.go

@@ -13,7 +13,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-func backupHub(dirPath string) error {
+func backupHub(hub *cwhub.Hub, dirPath string) error {
 	var err error
 	var itemDirectory string
 	var upstreamParsers []string
@@ -22,7 +22,7 @@ func backupHub(dirPath string) error {
 		clog := log.WithFields(log.Fields{
 			"type": itemType,
 		})
-		itemMap := cwhub.GetItemMap(itemType)
+		itemMap := hub.GetItemMap(itemType)
 		if itemMap == nil {
 			clog.Infof("No %s to backup.", itemType)
 			continue
@@ -189,7 +189,12 @@ func backupConfigToDirectory(dirPath string) error {
 		log.Infof("Saved profiles to %s", backupProfiles)
 	}
 
-	if err = backupHub(dirPath); err != nil {
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
+	if err = backupHub(hub, dirPath); err != nil {
 		return fmt.Errorf("failed to backup hub config: %s", err)
 	}
 
@@ -197,7 +202,7 @@ func backupConfigToDirectory(dirPath string) error {
 }
 
 func runConfigBackup(cmd *cobra.Command, args []string) error {
-	if err := require.Hub(csConfig); err != nil {
+	if _, err := require.Hub(csConfig); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/config_restore.go

@@ -23,23 +23,28 @@ type OldAPICfg struct {
 
 // it's a rip of the cli version, but in silent-mode
 func silentInstallItem(name string, obtype string) (string, error) {
-	var item = cwhub.GetItem(obtype, name)
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return "", err
+	}
+
+	var item = hub.GetItem(obtype, name)
 	if item == nil {
 		return "", fmt.Errorf("error retrieving item")
 	}
-	err := cwhub.DownloadLatest(csConfig.Hub, item, false, false)
+	err = hub.DownloadLatest(item, false, false)
 	if err != nil {
 		return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
 	}
-	if err := cwhub.AddItem(obtype, *item); err != nil {
+	if err := hub.AddItem(obtype, *item); err != nil {
 		return "", err
 	}
 
-	err = cwhub.EnableItem(csConfig.Hub, item)
+	err = hub.EnableItem(item)
 	if err != nil {
 		return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
 	}
-	if err := cwhub.AddItem(obtype, *item); err != nil {
+	if err := hub.AddItem(obtype, *item); err != nil {
 		return "", err
 	}
 	return fmt.Sprintf("Enabled %s", item.Name), nil
@@ -292,7 +297,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := require.Hub(csConfig); err != nil {
+	if _, err := require.Hub(csConfig); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/config_show.go

@@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
 cscli:
   - Output                  : {{.Cscli.Output}}
   - Hub Branch              : {{.Cscli.HubBranch}}
-  - Hub Folder              : {{.Cscli.HubDir}}
 {{- end }}
 
 {{- if .API }}

cmd/crowdsec-cli/console.go

@@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba
 				return fmt.Errorf("could not parse CAPI URL: %s", err)
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig)
+			if err != nil {
 				return err
 			}
 
-			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+			scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 			if err != nil {
 				return fmt.Errorf("failed to get installed scenarios: %s", err)
 			}

cmd/crowdsec-cli/hub.go

@@ -50,12 +50,13 @@ func runHubList(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err = require.Hub(csConfig); err != nil {
+	hub, err := require.Hub(csConfig)
+	if err != nil {
 		return err
 	}
 
 	// use LocalSync to get warnings about tainted / outdated items
-	warn, _ := cwhub.LocalSync(csConfig.Hub)
+	warn, _ := hub.LocalSync()
 	for _, v := range warn {
 		log.Info(v)
 	}
@@ -88,19 +89,24 @@ func NewHubListCmd() *cobra.Command {
 }
 
 func runHubUpdate(cmd *cobra.Command, args []string) error {
-	if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
+	cwhub.SetHubBranch()
+
+	// don't use require.Hub because if there is no index file, it would fail
+
+	hub, err := cwhub.InitHubUpdate(csConfig.Hub)
+	if err != nil {
 		if !errors.Is(err, cwhub.ErrIndexNotFound) {
 			return fmt.Errorf("failed to get Hub index : %w", err)
 		}
 		log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
 		cwhub.HubBranch = "master"
-		if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
+		if hub, err = cwhub.InitHubUpdate(csConfig.Hub); err != nil {
 			return fmt.Errorf("failed to get Hub index after retry: %w", err)
 		}
 	}
 
 	// use LocalSync to get warnings about tainted / outdated items
-	warn, _ := cwhub.LocalSync(csConfig.Hub)
+	warn, _ := hub.LocalSync()
 	for _, v := range warn {
 		log.Info(v)
 	}
@@ -122,8 +128,6 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 				return fmt.Errorf("you must configure cli before interacting with hub")
 			}
 
-			cwhub.SetHubBranch()
-
 			return nil
 		},
 		RunE: runHubUpdate,
@@ -140,27 +144,28 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := require.Hub(csConfig); err != nil {
+	hub, err := require.Hub(csConfig)
+	if err != nil {
 		return err
 	}
 
 	log.Infof("Upgrading collections")
-	if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil {
+	if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil {
 		return err
 	}
 
 	log.Infof("Upgrading parsers")
-	if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil {
+	if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil {
 		return err
 	}
 
 	log.Infof("Upgrading scenarios")
-	if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil {
+	if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil {
 		return err
 	}
 
 	log.Infof("Upgrading postoverflows")
-	if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
+	if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/item_metrics.go

@@ -18,7 +18,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func ShowMetrics(hubItem *cwhub.Item) {
+func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) {
 	switch hubItem.Type {
 	case cwhub.PARSERS:
 		metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
@@ -36,11 +36,11 @@ func ShowMetrics(hubItem *cwhub.Item) {
 			scenarioMetricsTable(color.Output, item, metrics)
 		}
 		for _, item := range hubItem.Collections {
-			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
+			hubItem = hub.GetItem(cwhub.COLLECTIONS, item)
 			if hubItem == nil {
 				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
 			}
-			ShowMetrics(hubItem)
+			ShowMetrics(hub, hubItem)
 		}
 	default:
 		log.Errorf("item of type '%s' is unknown", hubItem.Type)

cmd/crowdsec-cli/item_suggest.go

@@ -33,7 +33,11 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
 	allItems := make([]string, 0)
 	nearestScore := 100
 	nearestItem := &cwhub.Item{}
-	hubItems := cwhub.GetItemMap(itemType)
+
+	// XXX: handle error
+	hub, _ := cwhub.GetHub()
+
+	hubItems := hub.GetItemMap(itemType)
 	for _, item := range hubItems {
 		allItems = append(allItems, item.Name)
 	}
@@ -42,19 +46,20 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
 		d := levenshtein.Distance(itemName, s, nil)
 		if d < nearestScore {
 			nearestScore = d
-			nearestItem = cwhub.GetItem(itemType, s)
+			nearestItem = hub.GetItem(itemType, s)
 		}
 	}
 	return nearestItem, nearestScore
 }
 
 func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
+	hub, err := require.Hub(csConfig)
+	if err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}
 
 	comp := make([]string, 0)
-	hubItems := cwhub.GetItemMap(itemType)
+	hubItems := hub.GetItemMap(itemType)
 	for _, item := range hubItems {
 		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
 			comp = append(comp, item.Name)
@@ -65,11 +70,12 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
 }
 
 func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
+	hub, err := require.Hub(csConfig)
+	if err != nil {
 		return nil, cobra.ShellCompDirectiveDefault
 	}
 
-	items, err := cwhub.GetInstalledItemsAsString(itemType)
+	items, err := hub.GetInstalledItemsAsString(itemType)
 	if err != nil {
 		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
 		return nil, cobra.ShellCompDirectiveDefault

cmd/crowdsec-cli/items.go

@@ -16,8 +16,8 @@ import (
 )
 
 
-func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) {
-	itemNames := cwhub.GetItemNames(itemType)
+func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) {
+	itemNames := hub.GetItemNames(itemType)
 
 	notExist := []string{}
 	if len(args) > 0 {
@@ -40,7 +40,7 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string,
 	if installedOnly {
 		installed := []string{}
 		for _, item := range itemNames {
-			if cwhub.GetItem(itemType, item).Installed {
+			if hub.GetItem(itemType, item).Installed {
 				installed = append(installed, item)
 			}
 		}
@@ -52,9 +52,15 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string,
 
 func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
 	var err error
+
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	items := make(map[string][]string)
 	for _, itemType := range itemTypes {
-		if items[itemType], err = selectItems(itemType, args, !all); err != nil {
+		if items[itemType], err = selectItems(hub, itemType, args, !all); err != nil {
 			return err
 		}
 	}
@@ -78,7 +84,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
 			// empty slice in case there are no items of this type
 			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
 			for i, itemName := range items[itemType] {
-				item := cwhub.GetItem(itemType, itemName)
+				item := hub.GetItem(itemType, itemName)
 				status, emo := item.Status()
 				hubStatus[itemType][i] = itemHubStatus{
 					Name:         item.Name,
@@ -112,7 +118,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
 		}
 		for _, itemType := range itemTypes {
 			for _, itemName := range items[itemType] {
-				item := cwhub.GetItem(itemType, itemName)
+				item := hub.GetItem(itemType, itemName)
 				status, _ := item.Status()
 				if item.LocalVersion == "" {
 					item.LocalVersion = "n/a"
@@ -138,15 +144,17 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
 }
 
 func InspectItem(name string, itemType string, noMetrics bool) error {
-	hubItem := cwhub.GetItem(itemType, name)
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
+	hubItem := hub.GetItem(itemType, name)
 	if hubItem == nil {
 		return fmt.Errorf("can't find '%s' in %s", name, itemType)
 	}
 
-	var (
-		b   []byte
-		err error
-	)
+	var b   []byte
 
 	switch csConfig.Cscli.Output {
 	case "human", "raw":
@@ -168,7 +176,7 @@ func InspectItem(name string, itemType string, noMetrics bool) error {
 	}
 
 	fmt.Printf("\nCurrent metrics: \n")
-	ShowMetrics(hubItem)
+	ShowMetrics(hub, hubItem)
 
 	return nil
 }

cmd/crowdsec-cli/lapi.go

@@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
 	}
 
-	if err := require.Hub(csConfig); err != nil {
+	hub, err := require.Hub(csConfig)
+	if err != nil {
 		log.Fatal(err)
 	}
 
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		log.Fatalf("failed to get scenarios : %s", err)
 	}
@@ -338,9 +339,9 @@ cscli lapi context detect crowdsecurity/sshd-logs
 				log.Fatalf("Failed to init expr helpers : %s", err)
 			}
 
-			// Populate cwhub package tools
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Fatalf("Failed to load hub index : %s", err)
+			_, err = require.Hub(csConfig)
+			if err != nil {
+				log.Fatal(err)
 			}
 
 			csParsers := parser.NewParsers()

cmd/crowdsec-cli/parsers.go

@@ -25,7 +25,7 @@ cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
 		Aliases:           []string{"parser"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
+			if _, err := require.Hub(csConfig); err != nil {
 				return err
 			}
 
@@ -66,8 +66,13 @@ func runParsersInstall(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	for _, name := range args {
-		t := cwhub.GetItem(cwhub.PARSERS, name)
+		t := hub.GetItem(cwhub.PARSERS, name)
 		if t == nil {
 			nearestItem, score := GetDistance(cwhub.PARSERS, name)
 			Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
@@ -75,7 +80,7 @@ func runParsersInstall(cmd *cobra.Command, args []string) error {
 			continue
 		}
 
-		if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, force, downloadOnly); err != nil {
+		if err := hub.InstallItem(name, cwhub.PARSERS, force, downloadOnly); err != nil {
 			if !ignoreError {
 				return fmt.Errorf("error while installing '%s': %w", name, err)
 			}
@@ -126,8 +131,13 @@ func runParsersRemove(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, force)
+		err := hub.RemoveMany(cwhub.PARSERS, "", all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -140,7 +150,7 @@ func runParsersRemove(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, force)
+		err := hub.RemoveMany(cwhub.PARSERS, name, all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -184,8 +194,13 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil {
 			return err
 		}
 		return nil
@@ -196,7 +211,7 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.PARSERS, name, force); err != nil {
 			return err
 		}
 	}

cmd/crowdsec-cli/postoverflows.go

@@ -25,7 +25,7 @@ cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
 		Aliases:           []string{"postoverflow"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
+			if _, err := require.Hub(csConfig); err != nil {
 				return err
 			}
 
@@ -66,8 +66,13 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	for _, name := range args {
-		t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name)
+		t := hub.GetItem(cwhub.POSTOVERFLOWS, name)
 		if t == nil {
 			nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name)
 			Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError)
@@ -75,7 +80,7 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error {
 			continue
 		}
 
-		if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil {
+		if err := hub.InstallItem(name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil {
 			if !ignoreError {
 				return fmt.Errorf("error while installing '%s': %w", name, err)
 			}
@@ -126,8 +131,13 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force)
+		err := hub.RemoveMany(cwhub.POSTOVERFLOWS, "", all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -140,7 +150,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force)
+		err := hub.RemoveMany(cwhub.POSTOVERFLOWS, name, all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -184,8 +194,13 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil {
 			return err
 		}
 		return nil
@@ -196,7 +211,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, name, force); err != nil {
 			return err
 		}
 	}

cmd/crowdsec-cli/require/require.go

@@ -64,16 +64,17 @@ func Notifications(c *csconfig.Config) error {
 	return nil
 }
 
-func Hub (c *csconfig.Config) error {
+func Hub (c *csconfig.Config) (*cwhub.Hub, error) {
 	if c.Hub == nil {
-		return fmt.Errorf("you must configure cli before interacting with hub")
+		return nil, fmt.Errorf("you must configure cli before interacting with hub")
 	}
 
 	cwhub.SetHubBranch()
 
-	if err := cwhub.GetHubIdx(c.Hub); err != nil {
-		return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
+	hub, err := cwhub.InitHub(c.Hub)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
 	}
 
-	return nil
+	return hub, nil
 }

cmd/crowdsec-cli/scenarios.go

@@ -25,7 +25,7 @@ cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
 		Aliases:           []string{"scenario"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
+			if _, err := require.Hub(csConfig); err != nil {
 				return err
 			}
 
@@ -66,8 +66,13 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	for _, name := range args {
-		t := cwhub.GetItem(cwhub.SCENARIOS, name)
+		t := hub.GetItem(cwhub.SCENARIOS, name)
 		if t == nil {
 			nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
 			Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
@@ -75,7 +80,7 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error {
 			continue
 		}
 
-		if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, force, downloadOnly); err != nil {
+		if err := hub.InstallItem(name, cwhub.SCENARIOS, force, downloadOnly); err != nil {
 			if !ignoreError {
 				return fmt.Errorf("error while installing '%s': %w", name, err)
 			}
@@ -126,8 +131,13 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, force)
+		err := hub.RemoveMany(cwhub.SCENARIOS, "", all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -140,7 +150,7 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, force)
+		err := hub.RemoveMany(cwhub.SCENARIOS, name, all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -184,8 +194,13 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
 	if all {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil {
 			return err
 		}
 		return nil
@@ -196,7 +211,7 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force); err != nil {
+		if err := hub.UpgradeConfig(cwhub.SCENARIOS, name, force); err != nil {
 			return err
 		}
 	}

cmd/crowdsec-cli/simulation.go

@@ -145,13 +145,14 @@ func NewSimulationEnableCmd() *cobra.Command {
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig)
+			if err != nil {
 				log.Fatal(err)
 			}
 
 			if len(args) > 0 {
 				for _, scenario := range args {
-					var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
+					var item = hub.GetItem(cwhub.SCENARIOS, scenario)
 					if item == nil {
 						log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
 						continue

cmd/crowdsec-cli/support.go

@@ -155,7 +155,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
 	return out.Bytes(), nil
 }
 
-func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
+func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
 	if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
 		return []byte("No agent credentials found, are we LAPI ?")
 	}
@@ -165,7 +165,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 	if err != nil {
 		return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
 	}
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
 	}
@@ -293,7 +293,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				skipAgent = true
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig)
+			if err != nil {
 				log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 				skipHub = true
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
@@ -356,7 +357,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
 					csConfig.API.Server.OnlineClient.Credentials.Password,
 					csConfig.API.Server.OnlineClient.Credentials.URL,
-					CAPIURLPrefix)
+					CAPIURLPrefix,
+					hub)
 			}
 
 			if !skipLAPI {
@@ -364,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
 					csConfig.API.Client.Credentials.Password,
 					csConfig.API.Client.Credentials.URL,
-					LAPIURLPrefix)
+					LAPIURLPrefix,
+					hub)
 				infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
 			}
 

cmd/crowdsec-cli/utils_table.go

@@ -16,8 +16,10 @@ func listHubItemTable(out io.Writer, title string, itemType string, itemNames []
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
+	hub, _ := cwhub.GetHub()
+
 	for itemName := range itemNames {
-		item := cwhub.GetItem(itemType, itemNames[itemName])
+		item := hub.GetItem(itemType, itemNames[itemName])
 		status, emo := item.Status()
 		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.LocalVersion, item.LocalPath)
 	}

cmd/crowdsec/crowdsec.go

@@ -23,8 +23,8 @@ import (
 func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	var err error
 
-	// Populate cwhub package tools
-	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
+	hub, err := cwhub.InitHub(cConfig.Hub)
+	if err != nil {
 		return nil, fmt.Errorf("while loading hub index: %w", err)
 	}
 
@@ -34,7 +34,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 		return nil, fmt.Errorf("while loading parsers: %w", err)
 	}
 
-	if err := LoadBuckets(cConfig); err != nil {
+	if err := LoadBuckets(cConfig, hub); err != nil {
 		return nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 

cmd/crowdsec/main.go

@@ -75,12 +75,12 @@ type Flags struct {
 
 type labelsMap map[string]string
 
-func LoadBuckets(cConfig *csconfig.Config) error {
+func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
 	var (
 		err   error
 		files []string
 	)
-	for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
+	for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
 		if hubScenarioItem.Installed {
 			files = append(files, hubScenarioItem.LocalPath)
 		}
@@ -88,7 +88,7 @@ func LoadBuckets(cConfig *csconfig.Config) error {
 	buckets = leakybucket.NewBuckets()
 
 	log.Infof("Loading %d scenario files", len(files))
-	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
+	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, cConfig.ConfigPaths.DataDir, files, &bucketsTomb, buckets, flags.OrderEvent)
 
 	if err != nil {
 		return fmt.Errorf("scenario loading failed: %v", err)

cmd/crowdsec/output.go

@@ -70,7 +70,12 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	var cache []types.RuntimeAlert
 	var cacheMutex sync.Mutex
 
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return err
+	}
+
+	scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		return fmt.Errorf("loading list of installed hub scenarios: %w", err)
 	}
@@ -93,7 +98,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 		URL:            apiURL,
 		PapiURL:        papiURL,
 		VersionPrefix:  "v1",
-		UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
+		UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
 	})
 	if err != nil {
 		return fmt.Errorf("new client api: %w", err)

pkg/csconfig/config.go

@@ -100,7 +100,7 @@ func NewDefaultConfig() *Config {
 	commonCfg := CommonCfg{
 		Daemonize: false,
 		LogMedia:  "stdout",
-		LogLevel:   &logLevel,
+		LogLevel:  &logLevel,
 	}
 	prometheus := PrometheusCfg{
 		Enabled: true,

pkg/csconfig/crowdsec_service.go

@@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct {
 	BucketStateDumpDir        string            `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
 	BucketsGCEnabled          bool              `yaml:"-"`                          // we need to garbage collect buckets when in forensic mode
 
-	HubDir             string              `yaml:"-"`
-	DataDir            string              `yaml:"-"`
-	ConfigDir          string              `yaml:"-"`
-	HubIndexFile       string              `yaml:"-"`
 	SimulationFilePath string              `yaml:"-"`
 	ContextToSend      map[string][]string `yaml:"-"`
 }
@@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error {
 		return fmt.Errorf("load error (simulation): %w", err)
 	}
 
-	c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
-	c.Crowdsec.DataDir = c.ConfigPaths.DataDir
-	c.Crowdsec.HubDir = c.ConfigPaths.HubDir
-	c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
-
 	if c.Crowdsec.ParserRoutinesCount <= 0 {
 		c.Crowdsec.ParserRoutinesCount = 1
 	}
@@ -149,7 +140,7 @@ func (c *Config) LoadCrowdsec() error {
 	fallback := false
 	if c.Crowdsec.ConsoleContextPath == "" {
 		// fallback to default config file
-		c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
+		c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
 		fallback = true
 	}
 

pkg/csconfig/crowdsec_service_test.go

@@ -54,11 +54,6 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        "",
 				ConsoleContextPath:        contextFileFullPath,
 				AcquisitionFilePath:       acquisFullPath,
-				ConfigDir:                 "./testdata",
-				DataDir:                   "./data",
-				HubDir:                    "./hub",
-				// XXX: need to ensure a default here
-				HubIndexFile:              "",
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -98,11 +93,6 @@ func TestLoadCrowdsec(t *testing.T) {
 				AcquisitionDirPath:        acquisDirFullPath,
 				AcquisitionFilePath:       acquisFullPath,
 				ConsoleContextPath:        contextFileFullPath,
-				ConfigDir:                 "./testdata",
-				// XXX: need to ensure a default here
-				HubIndexFile:              "",
-				DataDir:                   "./data",
-				HubDir:                    "./hub",
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,
 				OutputRoutinesCount:       1,
@@ -139,11 +129,6 @@ func TestLoadCrowdsec(t *testing.T) {
 				Enable:                    ptr.Of(true),
 				AcquisitionDirPath:        "",
 				AcquisitionFilePath:       "",
-				ConfigDir:                 "./testdata",
-				// XXX: need to ensure a default here
-				HubIndexFile:              "",
-				DataDir:                   "./data",
-				HubDir:                    "./hub",
 				ConsoleContextPath:        contextFileFullPath,
 				BucketsRoutinesCount:      1,
 				ParserRoutinesCount:       1,

pkg/csconfig/cscli.go

@@ -11,10 +11,7 @@ type CscliCfg struct {
 	HubBranch          string            `yaml:"hub_branch"`
 	SimulationConfig   *SimulationConfig `yaml:"-"`
 	DbConfig           *DatabaseCfg      `yaml:"-"`
-	HubDir             string            `yaml:"-"`
-	DataDir            string            `yaml:"-"`
-	ConfigDir          string            `yaml:"-"`
-	HubIndexFile       string            `yaml:"-"`
+
 	SimulationFilePath string            `yaml:"-"`
 	PrometheusUrl      string            `yaml:"prometheus_uri"`
 }
@@ -23,10 +20,8 @@ func (c *Config) loadCSCLI() error {
 	if c.Cscli == nil {
 		c.Cscli = &CscliCfg{}
 	}
-	c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
-	c.Cscli.DataDir = c.ConfigPaths.DataDir
-	c.Cscli.HubDir = c.ConfigPaths.HubDir
-	c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
+
+	// XXX: HubBranch default should be set here and fed to HubCfg?
 
 	if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
 		c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)

pkg/csconfig/cscli_test.go

@@ -32,10 +32,6 @@ func TestLoadCSCLI(t *testing.T) {
 				},
 			},
 			expected: &CscliCfg{
-				ConfigDir:     "./testdata",
-				DataDir:       "./data",
-				HubDir:        "./hub",
-				HubIndexFile:  "./hub/.index.json",
 				PrometheusUrl: "http://127.0.0.1:6060/metrics",
 			},
 		},

pkg/csconfig/hub.go

@@ -9,6 +9,9 @@ type HubCfg struct {
 }
 
 func (c *Config) loadHub() error {
+
+	// XXX: HubBranch too -- from cscli or chooseHubBranch() ?
+
 	c.Hub = &HubCfg{
 		HubIndexFile:   c.ConfigPaths.HubIndexFile,
 		HubDir:         c.ConfigPaths.HubDir,

pkg/cwhub/cwhub.go

@@ -15,7 +15,6 @@ import (
 	"golang.org/x/mod/semver"
 )
 
-
 var (
 	ErrMissingReference = errors.New("Reference(s) missing in collection")
 
@@ -40,7 +39,7 @@ type Item struct {
 	FileName             string   `json:"file_name,omitempty"`                                                      // the filename, ie. apache2-logs.yaml
 	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .index.json
 	Author               string   `json:"author,omitempty"`                                                         // as seen in .index.json
-	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`              // as seen in .index.json
+	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`             // as seen in .index.json
 	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
 
 	// remote (hub) info
@@ -112,9 +111,16 @@ func (i *Item) versionStatus() int {
 	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
 }
 
+// validPath returns true if the (relative) path is allowed for the item
+// dirNmae: the directory name (ie. crowdsecurity)
+// fileName: the filename (ie. apache2-logs.yaml)
+func (i *Item) validPath(dirName, fileName string) bool {
+	return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
+}
+
 // GetItemMap returns the map of items for a given type
-func GetItemMap(itemType string) map[string]Item {
-	m, ok := hubIdx.Items[itemType]
+func (h *Hub) GetItemMap(itemType string) map[string]Item {
+	m, ok := h.Items[itemType]
 	if !ok {
 		return nil
 	}
@@ -123,6 +129,7 @@ func GetItemMap(itemType string) map[string]Item {
 }
 
 // itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
+// XXX: only used by leakybucket manager
 func itemKey(itemPath string) (string, error) {
 	f, err := os.Lstat(itemPath)
 	if err != nil {
@@ -150,13 +157,13 @@ func itemKey(itemPath string) (string, error) {
 }
 
 // GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
-func GetItemByPath(itemType string, itemPath string) (*Item, error) {
+func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
 	itemKey, err := itemKey(itemPath)
 	if err != nil {
 		return nil, err
 	}
 
-	m := GetItemMap(itemType)
+	m := h.GetItemMap(itemType)
 	if m == nil {
 		return nil, fmt.Errorf("item type %s doesn't exist", itemType)
 	}
@@ -170,19 +177,20 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) {
 }
 
 // GetItem returns the item from hub based on its type and full name (author/name)
-func GetItem(itemType string, itemName string) *Item {
-	if m, ok := GetItemMap(itemType)[itemName]; ok {
-		return &m
+func (h *Hub) GetItem(itemType string, itemName string) *Item {
+	m, ok := h.GetItemMap(itemType)[itemName]
+	if !ok {
+		return nil
 	}
 
-	return nil
+	return &m
 }
 
 // GetItemNames returns the list of item (full) names for a given type
 // ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
 // The names can be used to retrieve the item with GetItem()
-func GetItemNames(itemType string) []string {
-	m := GetItemMap(itemType)
+func (h *Hub) GetItemNames(itemType string) []string {
+	m := h.GetItemMap(itemType)
 	if m == nil {
 		return nil
 	}
@@ -196,10 +204,10 @@ func GetItemNames(itemType string) []string {
 }
 
 // AddItem adds an item to the hub index
-func AddItem(itemType string, item Item) error {
+func (h *Hub) AddItem(itemType string, item Item) error {
 	for _, itype := range ItemTypes {
 		if itype == itemType {
-			hubIdx.Items[itemType][item.Name] = item
+			h.Items[itemType][item.Name] = item
 			return nil
 		}
 	}
@@ -208,8 +216,8 @@ func AddItem(itemType string, item Item) error {
 }
 
 // GetInstalledItems returns the list of installed items
-func GetInstalledItems(itemType string) ([]Item, error) {
-	items, ok := hubIdx.Items[itemType]
+func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) {
+	items, ok := h.Items[itemType]
 	if !ok {
 		return nil, fmt.Errorf("no %s in hubIdx", itemType)
 	}
@@ -226,8 +234,8 @@ func GetInstalledItems(itemType string) ([]Item, error) {
 }
 
 // GetInstalledItemsAsString returns the names of the installed items
-func GetInstalledItemsAsString(itemType string) ([]string, error) {
-	items, err := GetInstalledItems(itemType)
+func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
+	items, err := h.GetInstalledItems(itemType)
 	if err != nil {
 		return nil, err
 	}

pkg/cwhub/cwhub_test.go

@@ -29,23 +29,15 @@ import (
 var responseByPath map[string]string
 
 func TestItemStatus(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
-
-	// DownloadHubIdx()
-	err := UpdateHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to download index")
-
-	err = GetHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to load hub index")
+	hub := envSetup(t)
 
 	// get existing map
-	x := GetItemMap(COLLECTIONS)
+	x := hub.GetItemMap(COLLECTIONS)
 	require.NotEmpty(t, x)
 
 	// Get item : good and bad
 	for k := range x {
-		item := GetItem(COLLECTIONS, k)
+		item := hub.GetItem(COLLECTIONS, k)
 		require.NotNil(t, item)
 
 		item.Installed = true
@@ -65,76 +57,101 @@ func TestItemStatus(t *testing.T) {
 		require.Equal(t, "disabled,local", txt)
 	}
 
-	DisplaySummary()
+	err := DisplaySummary()
+	require.NoError(t, err)
 }
 
 func TestGetters(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
-
-	// DownloadHubIdx()
-	err := UpdateHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to download index")
-
-	err = GetHubIdx(cfg.Hub)
-	require.NoError(t, err, "failed to load hub index")
+	hub := envSetup(t)
 
 	// get non existing map
-	empty := GetItemMap("ratata")
+	empty := hub.GetItemMap("ratata")
 	require.Nil(t, empty)
 
 	// get existing map
-	x := GetItemMap(COLLECTIONS)
+	x := hub.GetItemMap(COLLECTIONS)
 	require.NotEmpty(t, x)
 
 	// Get item : good and bad
 	for k := range x {
-		empty := GetItem(COLLECTIONS, k+"nope")
+		empty := hub.GetItem(COLLECTIONS, k+"nope")
 		require.Nil(t, empty)
 
-		item := GetItem(COLLECTIONS, k)
+		item := hub.GetItem(COLLECTIONS, k)
 		require.NotNil(t, item)
 
 		// Add item and get it
 		item.Name += "nope"
-		err := AddItem(COLLECTIONS, *item)
+		err := hub.AddItem(COLLECTIONS, *item)
 		require.NoError(t, err)
 
-		newitem := GetItem(COLLECTIONS, item.Name)
+		newitem := hub.GetItem(COLLECTIONS, item.Name)
 		require.NotNil(t, newitem)
 
-		err = AddItem("ratata", *item)
+		err = hub.AddItem("ratata", *item)
 		cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
 	}
 }
 
 func TestIndexDownload(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+	hub := envSetup(t)
 
-	// DownloadHubIdx()
-	err := UpdateHubIdx(cfg.Hub)
+	_, err := InitHubUpdate(hub.cfg)
 	require.NoError(t, err, "failed to download index")
 
-	err = GetHubIdx(cfg.Hub)
+	_, err = GetHub()
 	require.NoError(t, err, "failed to load hub index")
 }
 
-func getTestCfg() *csconfig.Config {
-	cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}}
-	cfg.Hub.InstallDir, _ = filepath.Abs("./install")
-	cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
-	cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
+// testHub initializes a temporary hub with an empty json file, optionally updating it
+func testHub(t *testing.T, update bool) *Hub {
+	tmpDir, err := os.MkdirTemp("", "testhub")
+	require.NoError(t, err)
+
+	hubCfg := &csconfig.HubCfg{
+		HubDir:         filepath.Join(tmpDir, "crowdsec", "hub"),
+		HubIndexFile:   filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"),
+		InstallDir:     filepath.Join(tmpDir, "crowdsec"),
+		InstallDataDir: filepath.Join(tmpDir, "installed-data"),
+	}
+
+	err = os.MkdirAll(hubCfg.HubDir, 0700)
+	require.NoError(t, err)
+
+	err = os.MkdirAll(hubCfg.InstallDir, 0700)
+	require.NoError(t, err)
+
+	err = os.MkdirAll(hubCfg.InstallDataDir, 0700)
+	require.NoError(t, err)
+
+	index, err := os.Create(hubCfg.HubIndexFile)
+	require.NoError(t, err)
+
+	_, err = index.WriteString(`{}`)
+	require.NoError(t, err)
 
-	return cfg
+	index.Close()
+
+	t.Cleanup(func() {
+		os.RemoveAll(tmpDir)
+	})
+
+	constructor := InitHub
+
+	if update {
+		constructor = InitHubUpdate
+	}
+
+	hub, err := constructor(hubCfg)
+	require.NoError(t, err)
+
+	return hub
 }
 
-func envSetup(t *testing.T) *csconfig.Config {
+func envSetup(t *testing.T) *Hub {
 	resetResponseByPath()
 	log.SetLevel(log.DebugLevel)
 
-	cfg := getTestCfg()
-
 	defaultTransport := http.DefaultClient.Transport
 
 	t.Cleanup(func() {
@@ -144,14 +161,7 @@ func envSetup(t *testing.T) *csconfig.Config {
 	// Mock the http client
 	http.DefaultClient.Transport = newMockTransport()
 
-	err := os.MkdirAll(cfg.Hub.InstallDir, 0700)
-	require.NoError(t, err)
-
-	err = os.MkdirAll(cfg.Hub.HubDir, 0700)
-	require.NoError(t, err)
-
-	err = UpdateHubIdx(cfg.Hub)
-	require.NoError(t, err)
+	hub := testHub(t, true)
 
 	// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
 	// 	log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
@@ -159,42 +169,33 @@ func envSetup(t *testing.T) *csconfig.Config {
 	// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
 	// 	log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
 	// }
-	return cfg
-}
-
-func envTearDown(cfg *csconfig.Config) {
-	if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
-		log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
-	}
-
-	if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
-		log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
-	}
+	return hub
 }
 
-func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
+func testInstallItem(hub *Hub, t *testing.T, item Item) {
 	// Install the parser
-	err := DownloadLatest(cfg, &item, false, false)
+
+	err := hub.DownloadLatest(&item, false, false)
 	require.NoError(t, err, "failed to download %s", item.Name)
 
-	_, err = LocalSync(cfg)
+	_, err = hub.LocalSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+	assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
 
-	err = EnableItem(cfg, &item)
+	err = hub.EnableItem(&item)
 	require.NoError(t, err, "failed to enable %s", item.Name)
 
-	_, err = LocalSync(cfg)
+	_, err = hub.LocalSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+	assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 }
 
-func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+func testTaintItem(hub *Hub, t *testing.T, item Item) {
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
 
 	f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
 	require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
@@ -205,54 +206,54 @@ func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
 	require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
 
 	// Local sync and check status
-	_, err = LocalSync(cfg)
+	_, err = hub.LocalSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
+	assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
 }
 
-func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
-	assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
+func testUpdateItem(hub *Hub, t *testing.T, item Item) {
+	assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
 
 	// Update it + check status
-	err := DownloadLatest(cfg, &item, true, true)
+	err := hub.DownloadLatest(&item, true, true)
 	require.NoError(t, err, "failed to update %s", item.Name)
 
 	// Local sync and check status
-	_, err = LocalSync(cfg)
+	_, err = hub.LocalSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+	assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
 }
 
-func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
-	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+func testDisableItem(hub *Hub, t *testing.T, item Item) {
+	assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 
 	// Remove
-	err := DisableItem(cfg, &item, false, false)
+	err := hub.DisableItem(&item, false, false)
 	require.NoError(t, err, "failed to disable %s", item.Name)
 
 	// Local sync and check status
-	warns, err := LocalSync(cfg)
+	warns, err := hub.LocalSync()
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, warns, "unexpected warnings : %+v", warns)
 
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
 
 	// Purge
-	err = DisableItem(cfg, &item, true, false)
+	err = hub.DisableItem(&item, true, false)
 	require.NoError(t, err, "failed to purge %s", item.Name)
 
 	// Local sync and check status
-	warns, err = LocalSync(cfg)
+	warns, err = hub.LocalSync()
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, warns, "unexpected warnings : %+v", warns)
 
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
 }
 
 func TestInstallParser(t *testing.T) {
@@ -265,20 +266,18 @@ func TestInstallParser(t *testing.T) {
 	 - check its status
 	 - remove it
 	*/
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+	hub := envSetup(t)
 
-	getHubIdxOrFail(t)
 	// map iteration is random by itself
-	for _, it := range hubIdx.Items[PARSERS] {
-		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx.Items[PARSERS][it.Name]
-		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx.Items[PARSERS][it.Name]
-		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx.Items[PARSERS][it.Name]
-		testDisableItem(cfg.Hub, t, it)
-		it = hubIdx.Items[PARSERS][it.Name]
+	for _, it := range hub.Items[PARSERS] {
+		testInstallItem(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testTaintItem(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testUpdateItem(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
+		testDisableItem(hub, t, it)
+		it = hub.Items[PARSERS][it.Name]
 
 		break
 	}
@@ -294,19 +293,17 @@ func TestInstallCollection(t *testing.T) {
 	 - check its status
 	 - remove it
 	*/
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+	hub := envSetup(t)
 
-	getHubIdxOrFail(t)
 	// map iteration is random by itself
-	for _, it := range hubIdx.Items[COLLECTIONS] {
-		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx.Items[COLLECTIONS][it.Name]
-		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx.Items[COLLECTIONS][it.Name]
-		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx.Items[COLLECTIONS][it.Name]
-		testDisableItem(cfg.Hub, t, it)
+	for _, it := range hub.Items[COLLECTIONS] {
+		testInstallItem(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testTaintItem(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testUpdateItem(hub, t, it)
+		it = hub.Items[COLLECTIONS][it.Name]
+		testDisableItem(hub, t, it)
 		break
 	}
 }

pkg/cwhub/download.go

@@ -19,31 +19,39 @@ import (
 
 var ErrIndexNotFound = fmt.Errorf("index not found")
 
-// UpdateHubIdx downloads the latest version of the index and updates the one in memory
-func UpdateHubIdx(hub *csconfig.HubCfg) error {
-	bidx, err := DownloadHubIdx(hub)
+// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk
+// It is used to inizialize the hub when there is no index file yet
+func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("no configuration found for hub")
+	}
+
+	bidx, err := DownloadHubIdx(cfg.HubIndexFile)
 	if err != nil {
-		return fmt.Errorf("failed to download index: %w", err)
+		return nil, fmt.Errorf("failed to download index: %w", err)
 	}
 
 	ret, err := ParseIndex(bidx)
 	if err != nil {
 		if !errors.Is(err, ErrMissingReference) {
-			return fmt.Errorf("failed to read index: %w", err)
+			return nil, fmt.Errorf("failed to read index: %w", err)
 		}
 	}
 
-	hubIdx = HubIndex{Items: ret}
+	theHub = &Hub{
+		Items: ret,
+		cfg:   cfg,
+	}
 
-	if _, err := LocalSync(hub); err != nil {
-		return fmt.Errorf("failed to sync: %w", err)
+	if _, err := theHub.LocalSync(); err != nil {
+		return nil, fmt.Errorf("failed to sync: %w", err)
 	}
 
-	return nil
+	return theHub, nil
 }
 
 // DownloadHubIdx downloads the latest version of the index and returns the content
-func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
+func DownloadHubIdx(indexPath string) ([]byte, error) {
 	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
 
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
@@ -70,7 +78,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
 		return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
 	}
 
-	oldContent, err := os.ReadFile(hub.HubIndexFile)
+	oldContent, err := os.ReadFile(indexPath)
 	if err != nil {
 		if !os.IsNotExist(err) {
 			log.Warningf("failed to read hub index: %s", err)
@@ -80,7 +88,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
 		// write it anyway, can't hurt
 	}
 
-	file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
 
 	if err != nil {
 		return nil, fmt.Errorf("while opening hub index file: %w", err)
@@ -92,13 +100,13 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
 		return nil, fmt.Errorf("while writing hub index file: %w", err)
 	}
 
-	log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
+	log.Infof("Wrote new %d bytes index to %s", wsize, indexPath)
 
 	return body, nil
 }
 
 // DownloadLatest will download the latest version of Item to the tdir directory
-func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error {
+func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error {
 	var err error
 
 	log.Debugf("Downloading %s %s", target.Type, target.Name)
@@ -109,7 +117,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
 			return nil
 		}
 
-		return DownloadItem(hub, target, overwrite)
+		return h.DownloadItem(target, overwrite)
 	}
 
 	// collection
@@ -117,7 +125,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
 	for idx, ptr := range tmp {
 		ptrtype := ItemTypes[idx]
 		for _, p := range ptr {
-			val, ok := hubIdx.Items[ptrtype][p]
+			val, ok := h.Items[ptrtype][p]
 			if !ok {
 				return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
 			}
@@ -132,7 +140,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
 			if ptrtype == COLLECTIONS {
 				log.Tracef("collection, recurse")
 
-				err = DownloadLatest(hub, &val, overwrite, updateOnly)
+				err = h.DownloadLatest(&val, overwrite, updateOnly)
 				if err != nil {
 					return fmt.Errorf("while downloading %s: %w", val.Name, err)
 				}
@@ -140,7 +148,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
 
 			downloaded := val.Downloaded
 
-			err = DownloadItem(hub, &val, overwrite)
+			err = h.DownloadItem(&val, overwrite)
 			if err != nil {
 				return fmt.Errorf("while downloading %s: %w", val.Name, err)
 			}
@@ -148,16 +156,16 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
 			// We need to enable an item when it has been added to a collection since latest release of the collection.
 			// We check if val.Downloaded is false because maybe the item has been disabled by the user.
 			if !val.Installed && !downloaded {
-				if err = EnableItem(hub, &val); err != nil {
+				if err = h.EnableItem(&val); err != nil {
 					return fmt.Errorf("enabling '%s': %w", val.Name, err)
 				}
 			}
 
-			hubIdx.Items[ptrtype][p] = val
+			h.Items[ptrtype][p] = val
 		}
 	}
 
-	err = DownloadItem(hub, target, overwrite)
+	err = h.DownloadItem(target, overwrite)
 	if err != nil {
 		return fmt.Errorf("failed to download item: %w", err)
 	}
@@ -165,8 +173,8 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
 	return nil
 }
 
-func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
-	tdir := hub.HubDir
+func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
+	tdir := h.cfg.HubDir
 
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
 	if !overwrite {
@@ -202,12 +210,12 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
 		return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
 	}
 
-	h := sha256.New()
-	if _, err = h.Write(body); err != nil {
+	hash := sha256.New()
+	if _, err = hash.Write(body); err != nil {
 		return fmt.Errorf("while hashing %s: %w", target.Name, err)
 	}
 
-	meow := fmt.Sprintf("%x", h.Sum(nil))
+	meow := fmt.Sprintf("%x", hash.Sum(nil))
 	if meow != target.Versions[target.Version].Digest {
 		log.Errorf("Downloaded version doesn't match index, please 'hub update'")
 		log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
@@ -263,18 +271,18 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
 	target.Tainted = false
 	target.UpToDate = true
 
-	if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
+	if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
 		return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
 	}
 
-	hubIdx.Items[target.Type][target.Name] = *target
+	h.Items[target.Type][target.Name] = *target
 
 	return nil
 }
 
 // DownloadDataIfNeeded downloads the data files for an item
-func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error {
-	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
+func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
+	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName)
 
 	itemFile, err := os.Open(itemFilePath)
 	if err != nil {
@@ -283,7 +291,7 @@ func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error {
 
 	defer itemFile.Close()
 
-	if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil {
+	if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil {
 		return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
 	}
 

pkg/cwhub/download_test.go

@@ -2,12 +2,11 @@ package cwhub
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 
 	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
 func TestDownloadHubIdx(t *testing.T) {
@@ -15,9 +14,18 @@ func TestDownloadHubIdx(t *testing.T) {
 	// bad url template
 	fmt.Println("Test 'bad URL'")
 
+	tmpIndex, err := os.CreateTemp("", "index.json")
+	if err != nil {
+		t.Fatalf("failed to create temp file : %s", err)
+	}
+
+	t.Cleanup(func() {
+		os.Remove(tmpIndex.Name())
+	})
+
 	RawFileURLTemplate = "x"
 
-	ret, err := DownloadHubIdx(&csconfig.HubCfg{})
+	ret, err := DownloadHubIdx(tmpIndex.Name())
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
 		log.Errorf("unexpected error %s", err)
 	}
@@ -29,7 +37,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = "https://baddomain/%s/%s"
 
-	ret, err = DownloadHubIdx(&csconfig.HubCfg{})
+	ret, err = DownloadHubIdx(tmpIndex.Name())
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
 		log.Errorf("unexpected error %s", err)
 	}
@@ -41,7 +49,7 @@ func TestDownloadHubIdx(t *testing.T) {
 
 	RawFileURLTemplate = back
 
-	ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"})
+	ret, err = DownloadHubIdx("/does/not/exist/index.json")
 	if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
 		log.Errorf("unexpected error %s", err)
 	}

pkg/cwhub/helpers.go

@@ -8,7 +8,6 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/mod/semver"
 
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 )
 
@@ -62,8 +61,8 @@ func SetHubBranch() {
 }
 
 // InstallItem installs an item from the hub
-func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error {
-	item := GetItem(itemType, name)
+func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error {
+	item := h.GetItem(itemType, name)
 	if item == nil {
 		return fmt.Errorf("unable to retrieve item: %s", name)
 	}
@@ -76,26 +75,26 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force
 		}
 	}
 
-	err := DownloadLatest(csConfig.Hub, item, force, true)
+	err := h.DownloadLatest(item, force, true)
 	if err != nil {
 		return fmt.Errorf("while downloading %s: %w", item.Name, err)
 	}
 
-	if err = AddItem(itemType, *item); err != nil {
+	if err = h.AddItem(itemType, *item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
 	if downloadOnly {
-		log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath))
+		log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.cfg.HubDir, item.RemotePath))
 		return nil
 	}
 
-	err = EnableItem(csConfig.Hub, item)
+	err = h.EnableItem(item)
 	if err != nil {
 		return fmt.Errorf("while enabling %s: %w", item.Name, err)
 	}
 
-	if err := AddItem(itemType, *item); err != nil {
+	if err := h.AddItem(itemType, *item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
@@ -105,20 +104,20 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force
 }
 
 // RemoveItem removes one - or all - the items from the hub
-func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error {
+func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, forceAction bool) error {
 	if name != "" {
-		item := GetItem(itemType, name)
+		item := h.GetItem(itemType, name)
 		if item == nil {
 			return fmt.Errorf("can't find '%s' in %s", name, itemType)
 		}
 
-		err := DisableItem(csConfig.Hub, item, purge, forceAction)
+		err := h.DisableItem(item, purge, forceAction)
 
 		if err != nil {
 			return fmt.Errorf("unable to disable %s: %w", item.Name, err)
 		}
 
-		if err = AddItem(itemType, *item); err != nil {
+		if err = h.AddItem(itemType, *item); err != nil {
 			return fmt.Errorf("unable to add %s: %w", item.Name, err)
 		}
 
@@ -132,17 +131,17 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
 	disabled := 0
 
 	// remove all
-	for _, v := range GetItemMap(itemType) {
+	for _, v := range h.GetItemMap(itemType) {
 		if !v.Installed {
 			continue
 		}
 
-		err := DisableItem(csConfig.Hub, &v, purge, forceAction)
+		err := h.DisableItem(&v, purge, forceAction)
 		if err != nil {
 			return fmt.Errorf("unable to disable %s: %w", v.Name, err)
 		}
 
-		if err := AddItem(itemType, v); err != nil {
+		if err := h.AddItem(itemType, v); err != nil {
 			return fmt.Errorf("unable to add %s: %w", v.Name, err)
 		}
 		disabled++
@@ -154,11 +153,11 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
 }
 
 // UpgradeConfig upgrades an item from the hub
-func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error {
+func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error {
 	updated := 0
 	found := false
 
-	for _, v := range GetItemMap(itemType) {
+	for _, v := range h.GetItemMap(itemType) {
 		if name != "" && name != v.Name {
 			continue
 		}
@@ -178,7 +177,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 		if v.UpToDate {
 			log.Infof("%s: up-to-date", v.Name)
 
-			if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil {
+			if err := h.DownloadDataIfNeeded(v, force); err != nil {
 				return fmt.Errorf("%s: download failed: %w", v.Name, err)
 			}
 
@@ -187,7 +186,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 			}
 		}
 
-		if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil {
+		if err := h.DownloadLatest(&v, force, true); err != nil {
 			return fmt.Errorf("%s: download failed: %w", v.Name, err)
 		}
 
@@ -205,7 +204,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
 			updated++
 		}
 
-		if err := AddItem(itemType, v); err != nil {
+		if err := h.AddItem(itemType, v); err != nil {
 			return fmt.Errorf("unable to add %s: %w", v.Name, err)
 		}
 	}

pkg/cwhub/helpers_test.go

@@ -4,165 +4,165 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
 // Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
 // We expect the new scenario to be installed
 func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+	hub := envSetup(t)
 
 	// fresh install of collection
-	getHubIdxOrFail(t)
+	hub = getHubOrFail(t, hub.cfg)
 
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
 
-	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
+	require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
 
 	// This is the scenario that gets added in next version of collection
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	pushUpdateToCollectionInHub()
 
-	if err := UpdateHubIdx(cfg.Hub); err != nil {
-		t.Fatalf("failed to download index : %s", err)
-	}
+	hub, err := InitHubUpdate(hub.cfg)
+	require.NoError(t, err, "failed to download index: %s", err)
 
-	getHubIdxOrFail(t)
+	hub = getHubOrFail(t, hub.cfg)
 
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
 
-	err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
 	require.NoError(t, err)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
-	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 }
 
 // Install a collection, disable a scenario.
 // Upgrade should install should not enable/download the disabled scenario.
 func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+	hub := envSetup(t)
 
 	// fresh install of collection
-	getHubIdxOrFail(t)
+	hub = getHubOrFail(t, hub.cfg)
 
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 
-	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
+	require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
-	err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
+	err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
 	require.NoError(t, err)
 
-	getHubIdxOrFail(t)
+	hub = getHubOrFail(t, hub.cfg)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
 
-	if err = UpdateHubIdx(cfg.Hub); err != nil {
-		t.Fatalf("failed to download index : %s", err)
-	}
+	hub, err = InitHubUpdate(hub.cfg)
+	require.NoError(t, err, "failed to download index: %s", err)
 
-	err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
 	require.NoError(t, err)
 
-	getHubIdxOrFail(t)
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	hub = getHubOrFail(t, hub.cfg)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 }
 
-func getHubIdxOrFail(t *testing.T) {
-	if err := GetHubIdx(getTestCfg().Hub); err != nil {
-		t.Fatalf("failed to load hub index")
-	}
+// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
+func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub {
+	hub, err := InitHub(hubCfg)
+	require.NoError(t, err, "failed to load hub index")
+	return hub
 }
 
 // Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
 // Upgrade should not enable/download the disabled scenario.
 // Upgrade should install and enable the newly added scenario.
 func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
-	cfg := envSetup(t)
-	defer envTearDown(cfg)
+	hub := envSetup(t)
 
 	// fresh install of collection
-	getHubIdxOrFail(t)
+	hub = getHubOrFail(t, hub.cfg)
 
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 
-	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
+	require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
-	err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
+	err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
 	require.NoError(t, err)
 
-	getHubIdxOrFail(t)
+	hub = getHubOrFail(t, hub.cfg)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
 	// we just removed. Nor should it install the newly added scenario
 	pushUpdateToCollectionInHub()
 
-	if err = UpdateHubIdx(cfg.Hub); err != nil {
-		t.Fatalf("failed to download index : %s", err)
-	}
+	hub, err = InitHubUpdate(hub.cfg)
+	require.NoError(t, err, "failed to download index: %s", err)
 
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	getHubIdxOrFail(t)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	hub = getHubOrFail(t, hub.cfg)
 
-	err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
+	err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
 	require.NoError(t, err)
 
-	getHubIdxOrFail(t)
-	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	hub = getHubOrFail(t, hub.cfg)
+	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 }
 
 func assertCollectionDepsInstalled(t *testing.T, collection string) {
 	t.Helper()
 
-	c := hubIdx.Items[COLLECTIONS][collection]
-	require.NoError(t, CollecDepsCheck(&c))
+	hub, err := GetHub()
+	require.NoError(t, err)
+
+	c := hub.Items[COLLECTIONS][collection]
+	require.NoError(t, hub.CollectDepsCheck(&c))
 }
 
 func pushUpdateToCollectionInHub() {

pkg/cwhub/hubindex.go → pkg/cwhub/hub.go

@@ -6,8 +6,9 @@ import (
 	"strings"
 
 	log "github.com/sirupsen/logrus"
-)
 
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+)
 
 const (
 	HubIndexFile = ".index.json"
@@ -20,25 +21,34 @@ const (
 )
 
 var (
-	// XXX: The order is important, as it is used to construct the
-	//      index tree in memory --> collections must be last
+	// XXX: The order is important, as it is used to range over sub-items in collections
 	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
-	hubIdx    = HubIndex{}
 )
 
-
 type HubItems map[string]map[string]Item
 
-// HubIndex represents the runtime status of the hub (parsed items, etc.)
-// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared
-type HubIndex struct {
-	Items HubItems
+// Hub represents the runtime status of the hub (parsed items, etc.)
+type Hub struct {
+	Items          HubItems
+	cfg            *csconfig.HubCfg
 	skippedLocal   int
 	skippedTainted int
 }
 
+var theHub *Hub
+
+// GetHub returns the hub singleton
+// it returns an error if it's not initialized to avoid nil dereference
+func GetHub() (*Hub, error) {
+	if theHub == nil {
+		return nil, fmt.Errorf("hub not initialized")
+	}
+
+	return theHub, nil
+}
+
 // displaySummary prints a total count of the hub items
-func (h HubIndex) displaySummary() {
+func (h Hub) displaySummary() {
 	msg := "Loaded: "
 	for itemType := range h.Items {
 		msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
@@ -52,8 +62,14 @@ func (h HubIndex) displaySummary() {
 
 // DisplaySummary prints a total count of the hub items.
 // It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton
-func DisplaySummary() {
-	hubIdx.displaySummary()
+// XXX: to be removed later
+func DisplaySummary() error {
+	hub, err := GetHub()
+	if err != nil {
+		return err
+	}
+	hub.displaySummary()
+	return nil
 }
 
 // ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections

pkg/cwhub/install.go

@@ -6,12 +6,10 @@ import (
 	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
-func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) {
-	itempath := hub.HubDir + "/" + target.RemotePath
+func (h *Hub) purgeItem(target Item) (Item, error) {
+	itempath := h.cfg.HubDir + "/" + target.RemotePath
 
 	// disable hub file
 	if err := os.Remove(itempath); err != nil {
@@ -20,19 +18,19 @@ func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) {
 
 	target.Downloaded = false
 	log.Infof("Removed source file [%s]: %s", target.Name, itempath)
-	hubIdx.Items[target.Type][target.Name] = target
+	h.Items[target.Type][target.Name] = target
 
 	return target, nil
 }
 
 // DisableItem to disable an item managed by the hub, removes the symlink if purge is true
-func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error {
+func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
 	var err error
 
 	// already disabled, noop unless purge
 	if !target.Installed {
 		if purge {
-			*target, err = purgeItem(hub, *target)
+			*target, err = h.purgeItem(*target)
 			if err != nil {
 				return err
 			}
@@ -54,7 +52,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
 		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
-				if val, ok := hubIdx.Items[ptrtype][p]; ok {
+				if val, ok := h.Items[ptrtype][p]; ok {
 					// check if the item doesn't belong to another collection before removing it
 					toRemove := true
 
@@ -66,7 +64,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
 					}
 
 					if toRemove {
-						err = DisableItem(hub, &val, purge, force)
+						err = h.DisableItem(&val, purge, force)
 						if err != nil {
 							return fmt.Errorf("while disabling %s: %w", p, err)
 						}
@@ -80,7 +78,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
 		}
 	}
 
-	syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
+	syml, err := filepath.Abs(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
 	if err != nil {
 		return err
 	}
@@ -103,7 +101,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
 			return fmt.Errorf("while reading symlink: %w", err)
 		}
 
-		absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
+		absPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
 		if err != nil {
 			return fmt.Errorf("while abs path: %w", err)
 		}
@@ -124,23 +122,23 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
 	target.Installed = false
 
 	if purge {
-		*target, err = purgeItem(hub, *target)
+		*target, err = h.purgeItem(*target)
 		if err != nil {
 			return err
 		}
 	}
 
-	hubIdx.Items[target.Type][target.Name] = *target
+	h.Items[target.Type][target.Name] = *target
 
 	return nil
 }
 
 // creates symlink between actual config file at hub.HubDir and hub.ConfigDir
 // Handles collections recursively
-func EnableItem(hub *csconfig.HubCfg, target *Item) error {
+func (h *Hub) EnableItem(target *Item) error {
 	var err error
 
-	parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
+	parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
 
 	// create directories if needed
 	if target.Installed {
@@ -172,12 +170,12 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error {
 		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
-				val, ok := hubIdx.Items[ptrtype][p]
+				val, ok := h.Items[ptrtype][p]
 				if !ok {
 					return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
 				}
 
-				err = EnableItem(hub, &val)
+				err = h.EnableItem(&val)
 				if err != nil {
 					return fmt.Errorf("while installing %s: %w", p, err)
 				}
@@ -192,7 +190,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error {
 	}
 
 	// hub.ConfigDir + target.RemotePath
-	srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
+	srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
 	if err != nil {
 		return fmt.Errorf("while getting source path: %w", err)
 	}
@@ -208,7 +206,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error {
 
 	log.Infof("Enabled %s : %s", target.Type, target.Name)
 	target.Installed = true
-	hubIdx.Items[target.Type][target.Name] = *target
+	h.Items[target.Type][target.Name] = *target
 
 	return nil
 }

pkg/cwhub/loader.go

@@ -19,10 +19,6 @@ func isYAMLFileName(path string) bool {
 	return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
 }
 
-func validItemFileName(vname string, fauthor string, fname string) bool {
-	return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml")
-}
-
 func handleSymlink(path string) (string, error) {
 	hubpath, err := os.Readlink(path)
 	if err != nil {
@@ -60,19 +56,6 @@ func getSHA256(filepath string) (string, error) {
 	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
 
-type Walker struct {
-	// the walk/parserVisit function can't receive extra args
-	hubdir     string
-	installdir string
-}
-
-func NewWalker(hub *csconfig.HubCfg) Walker {
-	return Walker{
-		hubdir:     hub.HubDir,
-		installdir: hub.InstallDir,
-	}
-}
-
 type itemFileInfo struct {
 	fname   string
 	stage   string
@@ -80,16 +63,19 @@ type itemFileInfo struct {
 	fauthor string
 }
 
-func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
+func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
 	ret := itemFileInfo{}
 	inhub := false
 
+	hubDir := h.cfg.HubDir
+	installDir := h.cfg.InstallDir
+
 	subs := strings.Split(path, string(os.PathSeparator))
 
-	log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir)
+	log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
 	log.Tracef("subs:%v", subs)
 	// we're in hub (~/.hub/hub/)
-	if strings.HasPrefix(path, w.hubdir) {
+	if strings.HasPrefix(path, hubDir) {
 		log.Tracef("in hub dir")
 
 		inhub = true
@@ -104,7 +90,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 		ret.fauthor = subs[len(subs)-2]
 		ret.stage = subs[len(subs)-3]
 		ret.ftype = subs[len(subs)-4]
-	} else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec/<type>/...
+	} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
 		log.Tracef("in install dir")
 		if len(subs) < 3 {
 			return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
@@ -118,7 +104,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 		ret.ftype = subs[len(subs)-3]
 		ret.fauthor = ""
 	} else {
-		return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir)
+		return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
 	}
 
 	log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
@@ -140,7 +126,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 	return ret, inhub, nil
 }
 
-func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
+func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 	var (
 		local   bool
 		hubpath string
@@ -166,7 +152,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 		return nil
 	}
 
-	info, inhub, err := w.getItemInfo(path)
+	info, inhub, err := h.getItemInfo(path)
 	if err != nil {
 		return err
 	}
@@ -197,12 +183,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 	// if it's not a symlink and not in hub, it's a local file, don't bother
 	if local && !inhub {
 		log.Tracef("%s is a local file, skip", path)
-		hubIdx.skippedLocal++
+		h.skippedLocal++
 		//	log.Infof("local scenario, skip.")
 
 		_, fileName := filepath.Split(path)
 
-		hubIdx.Items[info.ftype][info.fname] = Item{
+		h.Items[info.ftype][info.fname] = Item{
 			Name:      info.fname,
 			Stage:     info.stage,
 			Installed: true,
@@ -221,7 +207,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 
 	match := false
 
-	for name, item := range hubIdx.Items[info.ftype] {
+	for name, item := range h.Items[info.ftype] {
 		log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
 
 		if info.fname != item.FileName {
@@ -241,12 +227,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 				continue
 			}
 
-			// wrong file
-			if !validItemFileName(item.Name, info.fauthor, info.fname) {
+			// not the item we're looking for
+			if !item.validPath(info.fauthor, info.fname) {
 				continue
 			}
 
-			if path == w.hubdir+"/"+item.RemotePath {
+			if path == h.cfg.HubDir+"/"+item.RemotePath {
 				log.Tracef("marking %s as downloaded", item.Name)
 				item.Downloaded = true
 			}
@@ -303,7 +289,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 		if !match {
 			log.Tracef("got tainted match for %s: %s", item.Name, path)
 
-			hubIdx.skippedTainted++
+			h.skippedTainted++
 			// the file and the stage is right, but the hash is wrong, it has been tainted by user
 			if !inhub {
 				item.LocalPath = path
@@ -316,7 +302,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 			item.LocalHash = sha
 		}
 
-		hubIdx.Items[info.ftype][name] = item
+		h.Items[info.ftype][name] = item
 
 		return nil
 	}
@@ -326,13 +312,13 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 	return nil
 }
 
-func CollecDepsCheck(v *Item) error {
-	if v.versionStatus() != 0 { // not up-to-date
-		log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
+func (h *Hub) CollectDepsCheck(v *Item) error {
+	if v.Type != COLLECTIONS {
 		return nil
 	}
 
-	if v.Type != COLLECTIONS {
+	if v.versionStatus() != 0 { // not up-to-date
+		log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
 		return nil
 	}
 
@@ -342,7 +328,7 @@ func CollecDepsCheck(v *Item) error {
 	for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
 		sliceType := ItemTypes[idx]
 		for _, subName := range itemSlice {
-			subItem, ok := hubIdx.Items[sliceType][subName]
+			subItem, ok := h.Items[sliceType][subName]
 			if !ok {
 				return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name)
 			}
@@ -356,7 +342,7 @@ func CollecDepsCheck(v *Item) error {
 			if subItem.Type == COLLECTIONS {
 				log.Tracef("collec, recurse.")
 
-				if err := CollecDepsCheck(&subItem); err != nil {
+				if err := h.CollectDepsCheck(&subItem); err != nil {
 					if subItem.Tainted {
 						v.Tainted = true
 					}
@@ -364,7 +350,7 @@ func CollecDepsCheck(v *Item) error {
 					return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
 				}
 
-				hubIdx.Items[sliceType][subName] = subItem
+				h.Items[sliceType][subName] = subItem
 			}
 
 			// propagate the state of sub-items to set
@@ -395,7 +381,7 @@ func CollecDepsCheck(v *Item) error {
 				subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
 			}
 
-			hubIdx.Items[sliceType][subName] = subItem
+			h.Items[sliceType][subName] = subItem
 
 			log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
 		}
@@ -404,7 +390,7 @@ func CollecDepsCheck(v *Item) error {
 	return nil
 }
 
-func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
+func (h *Hub) SyncDir(dir string) ([]string, error) {
 	warnings := []string{}
 
 	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
@@ -414,13 +400,13 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
 			log.Errorf("failed %s : %s", cpath, err)
 		}
 
-		err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit)
+		err = filepath.WalkDir(cpath, h.itemVisit)
 		if err != nil {
 			return warnings, err
 		}
 	}
 
-	for name, item := range hubIdx.Items[COLLECTIONS] {
+	for name, item := range h.Items[COLLECTIONS] {
 		if !item.Installed {
 			continue
 		}
@@ -428,9 +414,9 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
 		vs := item.versionStatus()
 		switch vs {
 		case 0: // latest
-			if err := CollecDepsCheck(&item); err != nil {
+			if err := h.CollectDepsCheck(&item); err != nil {
 				warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
-				hubIdx.Items[COLLECTIONS][name] = item
+				h.Items[COLLECTIONS][name] = item
 			}
 		case 1: // not up-to-date
 			warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
@@ -445,51 +431,55 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
 }
 
 // Updates the info from HubInit() with the local state
-func LocalSync(hub *csconfig.HubCfg) ([]string, error) {
-	hubIdx.skippedLocal = 0
-	hubIdx.skippedTainted = 0
+func (h *Hub) LocalSync() ([]string, error) {
+	h.skippedLocal = 0
+	h.skippedTainted = 0
 
-	warnings, err := SyncDir(hub, hub.InstallDir)
+	warnings, err := h.SyncDir(h.cfg.InstallDir)
 	if err != nil {
-		return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err)
+		return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.InstallDir, err)
 	}
 
-	_, err = SyncDir(hub, hub.HubDir)
+	_, err = h.SyncDir(h.cfg.HubDir)
 	if err != nil {
-		return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err)
+		return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.HubDir, err)
 	}
 
 	return warnings, nil
 }
 
-func GetHubIdx(hub *csconfig.HubCfg) error {
-	if hub == nil {
-		return fmt.Errorf("no configuration found for hub")
+// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use
+func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
+	if cfg == nil {
+		return nil, fmt.Errorf("no configuration found for hub")
 	}
 
-	log.Debugf("loading hub idx %s", hub.HubIndexFile)
+	log.Debugf("loading hub idx %s", cfg.HubIndexFile)
 
-	bidx, err := os.ReadFile(hub.HubIndexFile)
+	bidx, err := os.ReadFile(cfg.HubIndexFile)
 	if err != nil {
-		return fmt.Errorf("unable to read index file: %w", err)
+		return nil, fmt.Errorf("unable to read index file: %w", err)
 	}
 
 	ret, err := ParseIndex(bidx)
 	if err != nil {
 		if !errors.Is(err, ErrMissingReference) {
-			return fmt.Errorf("unable to load existing index: %w", err)
+			return nil, fmt.Errorf("unable to load existing index: %w", err)
 		}
 
 		// XXX: why the error check if we bail out anyway?
-		return err
+		return nil, err
 	}
 
-	hubIdx = HubIndex{Items: ret}
+	theHub = &Hub{
+		Items: ret,
+		cfg:   cfg,
+	}
 
-	_, err = LocalSync(hub)
+	_, err = theHub.LocalSync()
 	if err != nil {
-		return fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
+		return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
 	}
 
-	return nil
+	return theHub, nil
 }

pkg/hubtest/hubtest.go

@@ -18,7 +18,7 @@ type HubTest struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *cwhub.HubIndex
+	HubIndex               *cwhub.Hub
 	Tests                  []*HubTestItem
 }
 
@@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 		TemplateConfigPath:     templateConfigFilePath,
 		TemplateProfilePath:    templateProfilePath,
 		TemplateSimulationPath: templateSimulationPath,
-		HubIndex:               &cwhub.HubIndex{Items: hubIndex},
+		HubIndex:               &cwhub.Hub{Items: hubIndex},
 	}, nil
 }
 

pkg/hubtest/hubtest_item.go

@@ -52,7 +52,7 @@ type HubTestItem struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *cwhub.HubIndex
+	HubIndex               *cwhub.Hub
 
 	Config *HubTestItemConfig
 
@@ -391,16 +391,16 @@ func (t *HubTestItem) InstallHub() error {
 	}
 
 	// load installed hub
-	err := cwhub.GetHubIdx(t.RuntimeHubConfig)
+	hub, err := cwhub.InitHub(t.RuntimeHubConfig)
 	if err != nil {
-		log.Fatalf("can't local sync the hub: %+v", err)
+		log.Fatal(err)
 	}
 
 	// install data for parsers if needed
-	ret := cwhub.GetItemMap(cwhub.PARSERS)
+	ret := hub.GetItemMap(cwhub.PARSERS)
 	for parserName, item := range ret {
 		if item.Installed {
-			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+			if err := hub.DownloadDataIfNeeded(item, true); err != nil {
 				return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
 			}
 			log.Debugf("parser '%s' installed successfully in runtime environment", parserName)
@@ -408,10 +408,10 @@ func (t *HubTestItem) InstallHub() error {
 	}
 
 	// install data for scenarios if needed
-	ret = cwhub.GetItemMap(cwhub.SCENARIOS)
+	ret = hub.GetItemMap(cwhub.SCENARIOS)
 	for scenarioName, item := range ret {
 		if item.Installed {
-			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+			if err := hub.DownloadDataIfNeeded(item, true); err != nil {
 				return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
 			}
 			log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName)
@@ -419,10 +419,10 @@ func (t *HubTestItem) InstallHub() error {
 	}
 
 	// install data for postoverflows if needed
-	ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS)
+	ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
 	for postoverflowName, item := range ret {
 		if item.Installed {
-			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
+			if err := hub.DownloadDataIfNeeded(item, true); err != nil {
 				return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
 			}
 			log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName)

pkg/leakybucket/buckets_test.go

@@ -8,12 +8,14 @@ import (
 	"html/template"
 	"io"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -33,7 +35,20 @@ func TestBucket(t *testing.T) {
 		envSetting = os.Getenv("TEST_ONLY")
 		tomb       = &tomb.Tomb{}
 	)
-	err := exprhelpers.Init(nil)
+
+	testdata := "./tests"
+
+	hubCfg := &csconfig.HubCfg{
+		HubDir: filepath.Join(testdata, "hub"),
+		HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
+	}
+
+	_, err := cwhub.InitHub(hubCfg)
+	if err != nil {
+		t.Fatalf("failed to init hub : %s", err)
+	}
+
+	err = exprhelpers.Init(nil)
 	if err != nil {
 		log.Fatalf("exprhelpers init failed: %s", err)
 	}
@@ -44,12 +59,15 @@ func TestBucket(t *testing.T) {
 		}
 	} else {
 		wg := new(sync.WaitGroup)
-		fds, err := os.ReadDir("./tests/")
+		fds, err := os.ReadDir(testdata)
 		if err != nil {
 			t.Fatalf("Unable to read test directory : %s", err)
 		}
 		for _, fd := range fds {
-			fname := "./tests/" + fd.Name()
+			if fd.Name() == "hub" {
+				continue
+			}
+			fname := filepath.Join(testdata, fd.Name())
 			log.Infof("Running test on %s", fname)
 			tomb.Go(func() error {
 				wg.Add(1)
@@ -112,10 +130,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
 		files = append(files, x.Filename)
 	}
 
-	cscfg := &csconfig.CrowdsecServiceCfg{
-		DataDir: "tests",
-	}
-	holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false)
+	cscfg := &csconfig.CrowdsecServiceCfg{}
+	holders, response, err := LoadBuckets(cscfg, "tests", files, tomb, buckets, false)
 	if err != nil {
 		t.Fatalf("failed loading bucket : %s", err)
 	}
@@ -123,7 +139,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
 		watchTomb(tomb)
 		return nil
 	})
-	if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) {
+	if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) {
 		return fmt.Errorf("tests from %s failed", dir)
 	}
 	return nil

pkg/leakybucket/manager_load.go

@@ -179,12 +179,17 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
 	return nil
 }
 
-func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
+func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
 	var (
 		ret      = []BucketFactory{}
 		response chan types.Event
 	)
 
+	hub, err := cwhub.GetHub()
+	if err != nil {
+		return nil, nil, err
+	}
+
 	response = make(chan types.Event, 1)
 	for _, f := range files {
 		log.Debugf("Loading '%s'", f)
@@ -212,7 +217,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
 				log.Tracef("End of yaml file")
 				break
 			}
-			bucketFactory.DataDir = cscfg.DataDir
+			bucketFactory.DataDir = dataDir
 			//check empty
 			if bucketFactory.Name == "" {
 				log.Errorf("Won't load nameless bucket")
@@ -235,7 +240,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
 			bucketFactory.Filename = filepath.Clean(f)
 			bucketFactory.BucketName = seed.Generate()
 			bucketFactory.ret = response
-			hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
+			hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
 			if err != nil {
 				log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
 			} else {

pkg/leakybucket/tests/hub/index.json

@@ -0,0 +1 @@
+{}

pkg/parser/unix_parser.go

@@ -64,8 +64,12 @@ func NewParsers() *Parsers {
 		StageFiles:      make([]Stagefile, 0),
 		PovfwStageFiles: make([]Stagefile, 0),
 	}
+
+	// XXX: handle error
+	hub, _ := cwhub.GetHub()
+
 	for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
-		for _, hubParserItem := range cwhub.GetItemMap(itemType) {
+		for _, hubParserItem := range hub.GetItemMap(itemType) {
 			if hubParserItem.Installed {
 				stagefile := Stagefile{
 					Filename: hubParserItem.LocalPath,
@@ -97,16 +101,16 @@ func NewParsers() *Parsers {
 func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
 	var err error
 
-	patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
+	patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/")
 	log.Infof("Loading grok library %s", patternsDir)
 	/* load base regexps for two grok parsers */
 	parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
-		"data": cConfig.Crowdsec.DataDir})
+		"data": cConfig.ConfigPaths.DataDir})
 	if err != nil {
 		return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
 	}
 	parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
-		"data": cConfig.Crowdsec.DataDir})
+		"data": cConfig.ConfigPaths.DataDir})
 	if err != nil {
 		return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
 	}
@@ -116,7 +120,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
 	*/
 	log.Infof("Loading enrich plugins")
 
-	parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
+	parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
 	if err != nil {
 		return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
 	}

pkg/setup/install.go

@@ -54,7 +54,8 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 
 	cwhub.SetHubBranch()
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+	hub, err := cwhub.InitHub(csConfig.Hub)
+	if err != nil {
 		return fmt.Errorf("getting hub index: %w", err)
 	}
 
@@ -75,7 +76,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
+				if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
 					return fmt.Errorf("while installing collection %s: %w", collection, err)
 				}
 			}
@@ -89,7 +90,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
+				if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
 					return fmt.Errorf("while installing parser %s: %w", parser, err)
 				}
 			}
@@ -103,7 +104,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
+				if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
 					return fmt.Errorf("while installing scenario %s: %w", scenario, err)
 				}
 			}
@@ -117,7 +118,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
+				if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
 					return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
 				}
 			}