HubIndex struct, comments, name changes (#2549)

* pkg/cwhub: rename PARSERS_OVFLW -> POSTOVERFLOWS
* mostly comments, some light cleanup
* move type hubtest.HubIndex -> cwhub.HubIndex
* move and rename LoadPkgIndex -> ParseIndex
* move displaySummary(), skippedLocal, skippedTainted to HubIndex struct

cmd/crowdsec-cli/config_backup.go

@@ -44,7 +44,7 @@ func backupHub(dirPath string) error {
 			//for the local/tainted ones, we backup the full file
 			if v.Tainted || v.Local || !v.UpToDate {
 				//we need to backup stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
 					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
 					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
 						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)

cmd/crowdsec-cli/config_restore.go

@@ -92,7 +92,7 @@ func restoreHub(dirPath string) error {
 			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
 				continue
 			}
-			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
+			if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
 				//we expect a stage here
 				if !file.IsDir() {
 					continue

cmd/crowdsec-cli/hub.go

@@ -63,7 +63,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
 	cwhub.DisplaySummary()
 
 	err = ListItems(color.Output, []string{
-		cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
+		cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS,
 	}, nil, true, false, all)
 	if err != nil {
 		return err
@@ -164,7 +164,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	log.Infof("Upgrading postoverflows")
-	if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force); err != nil {
+	if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/postoverflows.go

@@ -67,15 +67,15 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
+		t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name)
 		if t == nil {
-			nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
-			Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
+			nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name)
+			Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError)
 
 			continue
 		}
 
-		if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, force, downloadOnly); err != nil {
+		if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil {
 			if !ignoreError {
 				return fmt.Errorf("error while installing '%s': %w", name, err)
 			}
@@ -95,7 +95,7 @@ func NewPostOverflowsInstallCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
+			return compAllItems(cwhub.POSTOVERFLOWS, args, toComplete)
 		},
 		RunE: runPostOverflowsInstall,
 	}
@@ -127,7 +127,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
 	}
 
 	if all {
-		err := cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, force)
+		err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -140,7 +140,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		err := cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, force)
+		err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force)
 		if err != nil {
 			return err
 		}
@@ -158,7 +158,7 @@ func NewPostOverflowsRemoveCmd() *cobra.Command {
 		Aliases:           []string{"delete"},
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
+			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
 		},
 		RunE: runPostOverflowsRemove,
 	}
@@ -185,7 +185,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	if all {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", force); err != nil {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
 			return err
 		}
 		return nil
@@ -196,7 +196,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, force); err != nil {
+		if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil {
 			return err
 		}
 	}
@@ -212,7 +212,7 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command {
 		Example:           `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
+			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
 		},
 		RunE: runPostOverflowUpgrade,
 	}
@@ -242,7 +242,7 @@ func runPostOverflowsInspect(cmd *cobra.Command, args []string) error {
 	}
 
 	for _, name := range args {
-		if err = InspectItem(name, cwhub.PARSERS_OVFLW, noMetrics); err != nil {
+		if err = InspectItem(name, cwhub.POSTOVERFLOWS, noMetrics); err != nil {
 			return err
 		}
 	}
@@ -259,13 +259,13 @@ func NewPostOverflowsInspectCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
+			return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete)
 		},
 		RunE: runPostOverflowsInspect,
 	}
 
 	flags := cmdPostOverflowsInspect.Flags()
-	// XXX: is this needed for postoverflows?
+
 	flags.StringP("url", "u", "", "Prometheus url")
 	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
 
@@ -280,7 +280,7 @@ func runPostOverflowsList(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err = ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all); err != nil {
+	if err = ListItems(color.Output, []string{cwhub.POSTOVERFLOWS}, args, false, true, all); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/support.go

@@ -333,7 +333,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 			if !skipHub {
 				infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
 				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
-				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
+				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.POSTOVERFLOWS)
 				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
 			}
 

pkg/cwhub/cwhub.go

@@ -12,31 +12,15 @@ import (
 
 	"github.com/enescakir/emoji"
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
 	"golang.org/x/mod/semver"
 )
 
-const (
-	HubIndexFile = ".index.json"
-
-	// managed item types
-	PARSERS       = "parsers"
-	PARSERS_OVFLW = "postoverflows"
-	SCENARIOS     = "scenarios"
-	COLLECTIONS   = "collections"
-)
 
 var (
-	ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS}
-
 	ErrMissingReference = errors.New("Reference(s) missing in collection")
 
-	// XXX: can we remove these globals?
-	skippedLocal       = 0
-	skippedTainted     = 0
 	RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
 	HubBranch          = "master"
-	hubIdx             map[string]map[string]Item
 )
 
 // ItemVersion is used to detect the version of a given item
@@ -47,16 +31,16 @@ type ItemVersion struct {
 	Deprecated bool   `json:"deprecated,omitempty"` // XXX: do we keep this?
 }
 
-// Item can be: parser, scenario, collection..
+// Item represents an object managed in the hub. It can be a parser, scenario, collection..
 type Item struct {
 	// descriptive info
 	Type                 string   `json:"type,omitempty"                   yaml:"type,omitempty"`                   // parser|postoverflows|scenario|collection(|enrich)
 	Stage                string   `json:"stage,omitempty"                  yaml:"stage,omitempty"`                  // Stage for parser|postoverflow: s00-raw/s01-...
-	Name                 string   `json:"name,omitempty"`                                                           // as seen in .config.json, usually "author/name"
+	Name                 string   `json:"name,omitempty"`                                                           // as seen in .index.json, usually "author/name"
 	FileName             string   `json:"file_name,omitempty"`                                                      // the filename, ie. apache2-logs.yaml
-	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .config.json
-	Author               string   `json:"author,omitempty"`                                                         // as seen in .config.json
-	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`             // as seen in .config.json
+	Description          string   `json:"description,omitempty"            yaml:"description,omitempty"`            // as seen in .index.json
+	Author               string   `json:"author,omitempty"`                                                         // as seen in .index.json
+	References           []string `json:"references,omitempty"             yaml:"references,omitempty"`              // as seen in .index.json
 	BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
 
 	// remote (hub) info
@@ -74,13 +58,15 @@ type Item struct {
 	Tainted      bool   `json:"tainted,omitempty"` // has it been locally modified
 	Local        bool   `json:"local,omitempty"`   // if it's a non versioned control one
 
-	// if it's a collection, it's not a single file
+	// if it's a collection, it can have sub items
 	Parsers       []string `json:"parsers,omitempty"       yaml:"parsers,omitempty"`
 	PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
 	Scenarios     []string `json:"scenarios,omitempty"     yaml:"scenarios,omitempty"`
 	Collections   []string `json:"collections,omitempty"   yaml:"collections,omitempty"`
 }
 
+// Status returns the status of the item as a string and an emoji
+// ie. "enabled,update-available" and emoji.Warning
 func (i *Item) Status() (string, emoji.Emoji) {
 	status := "disabled"
 	ok := false
@@ -126,8 +112,9 @@ func (i *Item) versionStatus() int {
 	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
 }
 
+// GetItemMap returns the map of items for a given type
 func GetItemMap(itemType string) map[string]Item {
-	m, ok := hubIdx[itemType]
+	m, ok := hubIdx.Items[itemType]
 	if !ok {
 		return nil
 	}
@@ -135,7 +122,7 @@ func GetItemMap(itemType string) map[string]Item {
 	return m
 }
 
-// Given a FileInfo, extract the map key. Follow a symlink if necessary
+// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
 func itemKey(itemPath string) (string, error) {
 	f, err := os.Lstat(itemPath)
 	if err != nil {
@@ -182,6 +169,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) {
 	return &v, nil
 }
 
+// GetItem returns the item from hub based on its type and full name (author/name)
 func GetItem(itemType string, itemName string) *Item {
 	if m, ok := GetItemMap(itemType)[itemName]; ok {
 		return &m
@@ -207,10 +195,11 @@ func GetItemNames(itemType string) []string {
 	return names
 }
 
+// AddItem adds an item to the hub index
 func AddItem(itemType string, item Item) error {
 	for _, itype := range ItemTypes {
 		if itype == itemType {
-			hubIdx[itemType][item.Name] = item
+			hubIdx.Items[itemType][item.Name] = item
 			return nil
 		}
 	}
@@ -218,17 +207,9 @@ func AddItem(itemType string, item Item) error {
 	return fmt.Errorf("ItemType %s is unknown", itemType)
 }
 
-func DisplaySummary() {
-	log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
-		len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
-
-	if skippedLocal > 0 || skippedTainted > 0 {
-		log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
-	}
-}
-
+// GetInstalledItems returns the list of installed items
 func GetInstalledItems(itemType string) ([]Item, error) {
-	items, ok := hubIdx[itemType]
+	items, ok := hubIdx.Items[itemType]
 	if !ok {
 		return nil, fmt.Errorf("no %s in hubIdx", itemType)
 	}
@@ -244,6 +225,7 @@ func GetInstalledItems(itemType string) ([]Item, error) {
 	return retItems, nil
 }
 
+// GetInstalledItemsAsString returns the names of the installed items
 func GetInstalledItemsAsString(itemType string) ([]string, error) {
 	items, err := GetInstalledItems(itemType)
 	if err != nil {

pkg/cwhub/cwhub_test.go

@@ -180,9 +180,9 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
 
 	err = EnableItem(cfg, &item)
 	require.NoError(t, err, "failed to enable %s", item.Name)
@@ -190,11 +190,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 }
 
 func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
 
 	f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
 	require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
@@ -208,11 +208,11 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
 }
 
 func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
 
 	// Update it + check status
 	err := DownloadLatest(cfg, &item, true, true)
@@ -222,12 +222,12 @@ func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	_, err = LocalSync(cfg)
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
 }
 
 func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
-	assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
 
 	// Remove
 	err := DisableItem(cfg, &item, false, false)
@@ -238,9 +238,9 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, warns, "unexpected warnings : %+v", warns)
 
-	assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
 
 	// Purge
 	err = DisableItem(cfg, &item, true, false)
@@ -251,8 +251,8 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, warns, "unexpected warnings : %+v", warns)
 
-	assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
-	assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
 }
 
 func TestInstallParser(t *testing.T) {
@@ -270,15 +270,15 @@ func TestInstallParser(t *testing.T) {
 
 	getHubIdxOrFail(t)
 	// map iteration is random by itself
-	for _, it := range hubIdx[PARSERS] {
+	for _, it := range hubIdx.Items[PARSERS] {
 		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
+		it = hubIdx.Items[PARSERS][it.Name]
 		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
+		it = hubIdx.Items[PARSERS][it.Name]
 		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
+		it = hubIdx.Items[PARSERS][it.Name]
 		testDisableItem(cfg.Hub, t, it)
-		it = hubIdx[PARSERS][it.Name]
+		it = hubIdx.Items[PARSERS][it.Name]
 
 		break
 	}
@@ -299,13 +299,13 @@ func TestInstallCollection(t *testing.T) {
 
 	getHubIdxOrFail(t)
 	// map iteration is random by itself
-	for _, it := range hubIdx[COLLECTIONS] {
+	for _, it := range hubIdx.Items[COLLECTIONS] {
 		testInstallItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
+		it = hubIdx.Items[COLLECTIONS][it.Name]
 		testTaintItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
+		it = hubIdx.Items[COLLECTIONS][it.Name]
 		testUpdateItem(cfg.Hub, t, it)
-		it = hubIdx[COLLECTIONS][it.Name]
+		it = hubIdx.Items[COLLECTIONS][it.Name]
 		testDisableItem(cfg.Hub, t, it)
 		break
 	}

pkg/cwhub/download.go

@@ -19,20 +19,21 @@ import (
 
 var ErrIndexNotFound = fmt.Errorf("index not found")
 
+// UpdateHubIdx downloads the latest version of the index and updates the one in memory
 func UpdateHubIdx(hub *csconfig.Hub) error {
 	bidx, err := DownloadHubIdx(hub)
 	if err != nil {
 		return fmt.Errorf("failed to download index: %w", err)
 	}
 
-	ret, err := LoadPkgIndex(bidx)
+	ret, err := ParseIndex(bidx)
 	if err != nil {
 		if !errors.Is(err, ErrMissingReference) {
 			return fmt.Errorf("failed to read index: %w", err)
 		}
 	}
 
-	hubIdx = ret
+	hubIdx = HubIndex{Items: ret}
 
 	if _, err := LocalSync(hub); err != nil {
 		return fmt.Errorf("failed to sync: %w", err)
@@ -41,6 +42,7 @@ func UpdateHubIdx(hub *csconfig.Hub) error {
 	return nil
 }
 
+// DownloadHubIdx downloads the latest version of the index and returns the content
 func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
 	log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
 
@@ -115,7 +117,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly
 	for idx, ptr := range tmp {
 		ptrtype := ItemTypes[idx]
 		for _, p := range ptr {
-			val, ok := hubIdx[ptrtype][p]
+			val, ok := hubIdx.Items[ptrtype][p]
 			if !ok {
 				return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
 			}
@@ -151,7 +153,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly
 				}
 			}
 
-			hubIdx[ptrtype][p] = val
+			hubIdx.Items[ptrtype][p] = val
 		}
 	}
 
@@ -265,11 +267,12 @@ func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
 		return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
 	}
 
-	hubIdx[target.Type][target.Name] = *target
+	hubIdx.Items[target.Type][target.Name] = *target
 
 	return nil
 }
 
+// DownloadDataIfNeeded downloads the data files for an item
 func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
 	itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
 
@@ -287,6 +290,7 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
 	return nil
 }
 
+// downloadData downloads the data files for an item
 func downloadData(dataFolder string, force bool, reader io.Reader) error {
 	var err error
 

pkg/cwhub/helpers.go

@@ -12,12 +12,12 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 )
 
-// pick a hub branch corresponding to the current crowdsec version.
+// chooseHubBranch returns the branch name to use for the hub
+// It can be "master" or branch corresponding to the current crowdsec version
 func chooseHubBranch() string {
 	latest, err := cwversion.Latest()
 	if err != nil {
 		log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
-		//lint:ignore nilerr
 		return "master"
 	}
 
@@ -61,8 +61,9 @@ func SetHubBranch() {
 	log.Debugf("Using branch '%s' for the hub", HubBranch)
 }
 
-func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
-	item := GetItem(obtype, name)
+// InstallItem installs an item from the hub
+func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error {
+	item := GetItem(itemType, name)
 	if item == nil {
 		return fmt.Errorf("unable to retrieve item: %s", name)
 	}
@@ -80,7 +81,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
 		return fmt.Errorf("while downloading %s: %w", item.Name, err)
 	}
 
-	if err = AddItem(obtype, *item); err != nil {
+	if err = AddItem(itemType, *item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
@@ -94,7 +95,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
 		return fmt.Errorf("while enabling %s: %w", item.Name, err)
 	}
 
-	if err := AddItem(obtype, *item); err != nil {
+	if err := AddItem(itemType, *item); err != nil {
 		return fmt.Errorf("while adding %s: %w", item.Name, err)
 	}
 
@@ -103,6 +104,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo
 	return nil
 }
 
+// RemoveItem removes one - or all - the items from the hub
 func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error {
 	if name != "" {
 		item := GetItem(itemType, name)
@@ -151,6 +153,7 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
 	return nil
 }
 
+// UpgradeConfig upgrades an item from the hub
 func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error {
 	updated := 0
 	found := false

pkg/cwhub/helpers_test.go

@@ -15,19 +15,19 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
 	// fresh install of collection
 	getHubIdxOrFail(t)
 
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
 
 	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
 
 	// This is the scenario that gets added in next version of collection
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
@@ -40,17 +40,17 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
 
 	getHubIdxOrFail(t)
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
 
 	err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
 	require.NoError(t, err)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 }
 
 // Install a collection, disable a scenario.
@@ -62,17 +62,17 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	// fresh install of collection
 	getHubIdxOrFail(t)
 
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 
 	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
 	err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
@@ -80,11 +80,11 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 
 	getHubIdxOrFail(t)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
 
 	if err = UpdateHubIdx(cfg.Hub); err != nil {
 		t.Fatalf("failed to download index : %s", err)
@@ -94,7 +94,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	require.NoError(t, err)
 
 	getHubIdxOrFail(t)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 }
 
 func getHubIdxOrFail(t *testing.T) {
@@ -113,17 +113,17 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
 	// fresh install of collection
 	getHubIdxOrFail(t)
 
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 
 	require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
 
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
-	require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
 
 	err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
@@ -131,12 +131,12 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
 
 	getHubIdxOrFail(t)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
-	require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
+	require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
@@ -147,21 +147,21 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
 		t.Fatalf("failed to download index : %s", err)
 	}
 
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
 	getHubIdxOrFail(t)
 
 	err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
 	require.NoError(t, err)
 
 	getHubIdxOrFail(t)
-	require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
-	require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
+	require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
+	require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
 }
 
 func assertCollectionDepsInstalled(t *testing.T, collection string) {
 	t.Helper()
 
-	c := hubIdx[COLLECTIONS][collection]
+	c := hubIdx.Items[COLLECTIONS][collection]
 	require.NoError(t, CollecDepsCheck(&c))
 }
 

pkg/cwhub/hubindex.go

@@ -0,0 +1,104 @@
+package cwhub
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+
+const (
+	HubIndexFile = ".index.json"
+
+	// managed item types
+	COLLECTIONS   = "collections"
+	PARSERS       = "parsers"
+	POSTOVERFLOWS = "postoverflows"
+	SCENARIOS     = "scenarios"
+)
+
+var (
+	// XXX: The order is important, as it is used to construct the
+	//      index tree in memory --> collections must be last
+	ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
+	hubIdx    = HubIndex{}
+)
+
+
+type HubItems map[string]map[string]Item
+
+type HubIndex struct {
+	Items HubItems
+	skippedLocal   int
+	skippedTainted int
+}
+
+// displaySummary prints a total count of the hub items
+func (h HubIndex) displaySummary() {
+	msg := "Loaded: "
+	for itemType := range h.Items {
+		msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
+	}
+	log.Info(strings.Trim(msg, ", "))
+
+	if h.skippedLocal > 0 || h.skippedTainted > 0 {
+		log.Infof("unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted)
+	}
+}
+
+// DisplaySummary prints a total count of the hub items.
+// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton
+func DisplaySummary() {
+	hubIdx.displaySummary()
+}
+
+// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections
+func ParseIndex(buff []byte) (HubItems, error) {
+	var (
+		RawIndex     HubItems
+		missingItems []string
+	)
+
+	if err := json.Unmarshal(buff, &RawIndex); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal index: %w", err)
+	}
+
+	log.Debugf("%d item types in hub index", len(ItemTypes))
+
+	// Iterate over the different types to complete the struct
+	for _, itemType := range ItemTypes {
+		log.Tracef("%s: %d items", itemType, len(RawIndex[itemType]))
+
+		for name, item := range RawIndex[itemType] {
+			item.Name = name
+			item.Type = itemType
+			x := strings.Split(item.RemotePath, "/")
+			item.FileName = x[len(x)-1]
+			RawIndex[itemType][name] = item
+
+			if itemType != COLLECTIONS {
+				continue
+			}
+
+			// if it's a collection, check its sub-items are present
+			// XXX should be done later
+			for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
+				ptrtype := ItemTypes[idx]
+				for _, p := range ptr {
+					if _, ok := RawIndex[ptrtype][p]; !ok {
+						log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
+						missingItems = append(missingItems, p)
+					}
+				}
+			}
+		}
+	}
+
+	if len(missingItems) > 0 {
+		return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
+	}
+
+	return RawIndex, nil
+}

pkg/cwhub/install.go

@@ -20,7 +20,7 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
 
 	target.Downloaded = false
 	log.Infof("Removed source file [%s]: %s", target.Name, itempath)
-	hubIdx[target.Type][target.Name] = target
+	hubIdx.Items[target.Type][target.Name] = target
 
 	return target, nil
 }
@@ -54,7 +54,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error
 		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
-				if val, ok := hubIdx[ptrtype][p]; ok {
+				if val, ok := hubIdx.Items[ptrtype][p]; ok {
 					// check if the item doesn't belong to another collection before removing it
 					toRemove := true
 
@@ -130,7 +130,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error
 		}
 	}
 
-	hubIdx[target.Type][target.Name] = *target
+	hubIdx.Items[target.Type][target.Name] = *target
 
 	return nil
 }
@@ -172,7 +172,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error {
 		for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
-				val, ok := hubIdx[ptrtype][p]
+				val, ok := hubIdx.Items[ptrtype][p]
 				if !ok {
 					return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
 				}
@@ -208,7 +208,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error {
 
 	log.Infof("Enabled %s : %s", target.Type, target.Name)
 	target.Installed = true
-	hubIdx[target.Type][target.Name] = *target
+	hubIdx.Items[target.Type][target.Name] = *target
 
 	return nil
 }

pkg/cwhub/loader.go

@@ -2,7 +2,6 @@ package cwhub
 
 import (
 	"crypto/sha256"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -131,8 +130,8 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 	} else if ret.stage == COLLECTIONS {
 		ret.ftype = COLLECTIONS
 		ret.stage = ""
-	} else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW {
-		// its a PARSER / PARSER_OVFLW with a stage
+	} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
+		// its a PARSER / POSTOVERFLOW with a stage
 		return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
 	}
 
@@ -141,7 +140,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
 	return ret, inhub, nil
 }
 
-func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
+func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
 	var (
 		local   bool
 		hubpath string
@@ -198,12 +197,12 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 	// if it's not a symlink and not in hub, it's a local file, don't bother
 	if local && !inhub {
 		log.Tracef("%s is a local file, skip", path)
-		skippedLocal++
+		hubIdx.skippedLocal++
 		//	log.Infof("local scenario, skip.")
 
 		_, fileName := filepath.Split(path)
 
-		hubIdx[info.ftype][info.fname] = Item{
+		hubIdx.Items[info.ftype][info.fname] = Item{
 			Name:      info.fname,
 			Stage:     info.stage,
 			Installed: true,
@@ -222,7 +221,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 
 	match := false
 
-	for name, item := range hubIdx[info.ftype] {
+	for name, item := range hubIdx.Items[info.ftype] {
 		log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
 
 		if info.fname != item.FileName {
@@ -304,7 +303,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 		if !match {
 			log.Tracef("got tainted match for %s: %s", item.Name, path)
 
-			skippedTainted++
+			hubIdx.skippedTainted++
 			// the file and the stage is right, but the hash is wrong, it has been tainted by user
 			if !inhub {
 				item.LocalPath = path
@@ -317,14 +316,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
 			item.LocalHash = sha
 		}
 
-		// update the entry if appropriate
-		// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
-		// 	fmt.Printf("Updating %s", k)
-		// 	hubIdx[ftype][k] = v
-		// } else if !inhub {
-
-		// } else if
-		hubIdx[info.ftype][name] = item
+		hubIdx.Items[info.ftype][name] = item
 
 		return nil
 	}
@@ -350,7 +342,7 @@ func CollecDepsCheck(v *Item) error {
 	for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
 		sliceType := ItemTypes[idx]
 		for _, subName := range itemSlice {
-			subItem, ok := hubIdx[sliceType][subName]
+			subItem, ok := hubIdx.Items[sliceType][subName]
 			if !ok {
 				return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name)
 			}
@@ -372,7 +364,7 @@ func CollecDepsCheck(v *Item) error {
 					return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
 				}
 
-				hubIdx[sliceType][subName] = subItem
+				hubIdx.Items[sliceType][subName] = subItem
 			}
 
 			// propagate the state of sub-items to set
@@ -403,7 +395,7 @@ func CollecDepsCheck(v *Item) error {
 				subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
 			}
 
-			hubIdx[sliceType][subName] = subItem
+			hubIdx.Items[sliceType][subName] = subItem
 
 			log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
 		}
@@ -415,20 +407,20 @@ func CollecDepsCheck(v *Item) error {
 func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
 	warnings := []string{}
 
-	// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
+	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
 	for _, scan := range ItemTypes {
 		cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
 		if err != nil {
 			log.Errorf("failed %s : %s", cpath, err)
 		}
 
-		err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit)
+		err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit)
 		if err != nil {
 			return warnings, err
 		}
 	}
 
-	for name, item := range hubIdx[COLLECTIONS] {
+	for name, item := range hubIdx.Items[COLLECTIONS] {
 		if !item.Installed {
 			continue
 		}
@@ -438,7 +430,7 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
 		case 0: // latest
 			if err := CollecDepsCheck(&item); err != nil {
 				warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
-				hubIdx[COLLECTIONS][name] = item
+				hubIdx.Items[COLLECTIONS][name] = item
 			}
 		case 1: // not up-to-date
 			warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
@@ -454,8 +446,8 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
 
 // Updates the info from HubInit() with the local state
 func LocalSync(hub *csconfig.Hub) ([]string, error) {
-	skippedLocal = 0
-	skippedTainted = 0
+	hubIdx.skippedLocal = 0
+	hubIdx.skippedTainted = 0
 
 	warnings, err := SyncDir(hub, hub.InstallDir)
 	if err != nil {
@@ -482,7 +474,7 @@ func GetHubIdx(hub *csconfig.Hub) error {
 		return fmt.Errorf("unable to read index file: %w", err)
 	}
 
-	ret, err := LoadPkgIndex(bidx)
+	ret, err := ParseIndex(bidx)
 	if err != nil {
 		if !errors.Is(err, ErrMissingReference) {
 			return fmt.Errorf("unable to load existing index: %w", err)
@@ -492,7 +484,7 @@ func GetHubIdx(hub *csconfig.Hub) error {
 		return err
 	}
 
-	hubIdx = ret
+	hubIdx = HubIndex{Items: ret}
 
 	_, err = LocalSync(hub)
 	if err != nil {
@@ -501,52 +493,3 @@ func GetHubIdx(hub *csconfig.Hub) error {
 
 	return nil
 }
-
-// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections
-func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
-	var (
-		RawIndex     map[string]map[string]Item
-		missingItems []string
-	)
-
-	if err := json.Unmarshal(buff, &RawIndex); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal index: %w", err)
-	}
-
-	log.Debugf("%d item types in hub index", len(ItemTypes))
-
-	// Iterate over the different types to complete the struct
-	for _, itemType := range ItemTypes {
-		log.Tracef("%d item", len(RawIndex[itemType]))
-
-		for name, item := range RawIndex[itemType] {
-			item.Name = name
-			item.Type = itemType
-			x := strings.Split(item.RemotePath, "/")
-			item.FileName = x[len(x)-1]
-			RawIndex[itemType][name] = item
-
-			if itemType != COLLECTIONS {
-				continue
-			}
-
-			// if it's a collection, check its sub-items are present
-			// XXX should be done later
-			for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
-				ptrtype := ItemTypes[idx]
-				for _, p := range ptr {
-					if _, ok := RawIndex[ptrtype][p]; !ok {
-						log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
-						missingItems = append(missingItems, p)
-					}
-				}
-			}
-		}
-	}
-
-	if len(missingItems) > 0 {
-		return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
-	}
-
-	return RawIndex, nil
-}

pkg/hubtest/coverage.go

@@ -27,12 +27,12 @@ type ScenarioCoverage struct {
 
 func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
 	var coverage []ParserCoverage
-	if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok {
+	if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
 		return coverage, fmt.Errorf("no parsers in hub index")
 	}
 	//populate from hub, iterate in alphabetical order
 	var pkeys []string
-	for pname := range h.HubIndex.Data[cwhub.PARSERS] {
+	for pname := range h.HubIndex.Items[cwhub.PARSERS] {
 		pkeys = append(pkeys, pname)
 	}
 	sort.Strings(pkeys)
@@ -100,12 +100,12 @@ func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
 
 func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) {
 	var coverage []ScenarioCoverage
-	if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok {
+	if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
 		return coverage, fmt.Errorf("no scenarios in hub index")
 	}
 	//populate from hub, iterate in alphabetical order
 	var pkeys []string
-	for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] {
+	for scenarioName := range h.HubIndex.Items[cwhub.SCENARIOS] {
 		pkeys = append(pkeys, scenarioName)
 	}
 	sort.Strings(pkeys)

pkg/hubtest/hubtest.go

@@ -18,7 +18,7 @@ type HubTest struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *HubIndex
+	HubIndex               *cwhub.HubIndex
 	Tests                  []*HubTestItem
 }
 
@@ -62,7 +62,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 	}
 
 	// load hub index
-	hubIndex, err := cwhub.LoadPkgIndex(bidx)
+	hubIndex, err := cwhub.ParseIndex(bidx)
 	if err != nil {
 		return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err)
 	}
@@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
 		TemplateConfigPath:     templateConfigFilePath,
 		TemplateProfilePath:    templateProfilePath,
 		TemplateSimulationPath: templateSimulationPath,
-		HubIndex:               &HubIndex{Data: hubIndex},
+		HubIndex:               &cwhub.HubIndex{Items: hubIndex},
 	}, nil
 }
 

pkg/hubtest/hubtest_item.go

@@ -25,10 +25,6 @@ type HubTestItemConfig struct {
 	OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
 }
 
-type HubIndex struct {
-	Data map[string]map[string]cwhub.Item
-}
-
 type HubTestItem struct {
 	Name string
 	Path string
@@ -56,7 +52,7 @@ type HubTestItem struct {
 	TemplateConfigPath     string
 	TemplateProfilePath    string
 	TemplateSimulationPath string
-	HubIndex               *HubIndex
+	HubIndex               *cwhub.HubIndex
 
 	Config *HubTestItemConfig
 
@@ -148,7 +144,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 		var parserDirDest string
-		if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok {
+		if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
 			parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
@@ -232,7 +228,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 		var scenarioDirDest string
-		if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok {
+		if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
 			scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
@@ -301,7 +297,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 		var postoverflowDirDest string
-		if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok {
+		if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
 			postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
@@ -423,7 +419,7 @@ func (t *HubTestItem) InstallHub() error {
 	}
 
 	// install data for postoverflows if needed
-	ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW)
+	ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS)
 	for postoverflowName, item := range ret {
 		if item.Installed {
 			if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {

pkg/parser/unix_parser.go

@@ -64,7 +64,7 @@ func NewParsers() *Parsers {
 		StageFiles:      make([]Stagefile, 0),
 		PovfwStageFiles: make([]Stagefile, 0),
 	}
-	for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
+	for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
 		for _, hubParserItem := range cwhub.GetItemMap(itemType) {
 			if hubParserItem.Installed {
 				stagefile := Stagefile{
@@ -74,7 +74,7 @@ func NewParsers() *Parsers {
 				if itemType == cwhub.PARSERS {
 					parsers.StageFiles = append(parsers.StageFiles, stagefile)
 				}
-				if itemType == cwhub.PARSERS_OVFLW {
+				if itemType == cwhub.POSTOVERFLOWS {
 					parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
 				}
 			}

pkg/setup/install.go

@@ -121,7 +121,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
 					continue
 				}
 
-				if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
+				if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
 					return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
 				}
 			}