command "cscli hub types" (#2632)

* Command "cscli hub types"; de-duplicate test/bin/preload-hub-items
* don't export Hub.Items -> hub.items

cmd/crowdsec-cli/hub.go

@@ -1,11 +1,13 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@@ -29,6 +31,7 @@ cscli hub upgrade`,
 	cmdHub.AddCommand(NewHubListCmd())
 	cmdHub.AddCommand(NewHubUpdateCmd())
 	cmdHub.AddCommand(NewHubUpgradeCmd())
+	cmdHub.AddCommand(NewHubTypesCmd())
 
 	return cmdHub
 }
@@ -172,3 +175,40 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 
 	return cmdHubUpgrade
 }
+
+func runHubTypes(cmd *cobra.Command, args []string) error {
+	switch csConfig.Cscli.Output {
+	case "human":
+		s, err := yaml.Marshal(cwhub.ItemTypes)
+		if err != nil {
+			return err
+		}
+		fmt.Print(string(s))
+	case "json":
+		jsonStr, err := json.Marshal(cwhub.ItemTypes)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(jsonStr))
+	case "raw":
+		for _, itemType := range cwhub.ItemTypes {
+			fmt.Println(itemType)
+		}
+	}
+	return nil
+}
+
+func NewHubTypesCmd() *cobra.Command {
+	cmdHubTypes := &cobra.Command{
+		Use:   "types",
+		Short: "List supported item types",
+		Long: `
+List the types of supported hub items.
+`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		RunE:              runHubTypes,
+	}
+
+	return cmdHubTypes
+}

cmd/crowdsec-cli/itemcommands.go

@@ -36,7 +36,7 @@ type hubItemType struct {
 
 var hubItemTypes = map[string]hubItemType{
 	"parsers": {
-		name:      "parsers",
+		name:      cwhub.PARSERS,
 		singular:  "parser",
 		oneOrMore: "parser(s)",
 		help: cmdHelp{
@@ -68,7 +68,7 @@ List only enabled parsers unless "-a" or names are specified.`,
 		},
 	},
 	"postoverflows": {
-		name:      "postoverflows",
+		name:      cwhub.POSTOVERFLOWS,
 		singular:  "postoverflow",
 		oneOrMore: "postoverflow(s)",
 		help: cmdHelp{
@@ -100,7 +100,7 @@ List only enabled postoverflows unless "-a" or names are specified.`,
 		},
 	},
 	"scenarios": {
-		name:      "scenarios",
+		name:      cwhub.SCENARIOS,
 		singular:  "scenario",
 		oneOrMore: "scenario(s)",
 		help: cmdHelp{
@@ -132,7 +132,7 @@ List only enabled scenarios unless "-a" or names are specified.`,
 		},
 	},
 	"collections": {
-		name:      "collections",
+		name:      cwhub.COLLECTIONS,
 		singular:  "collection",
 		oneOrMore: "collection(s)",
 		help: cmdHelp{

pkg/cwhub/hub.go

@@ -9,13 +9,14 @@ import (
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+	"slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
 // Hub is the main structure for the package.
 type Hub struct {
-	Items    HubItems // Items read from HubDir and InstallDir
+	items    HubItems // Items read from HubDir and InstallDir
 	local    *csconfig.LocalHubCfg
 	remote   *RemoteHubCfg
 	Warnings []string // Warnings encountered during sync
@@ -65,7 +66,7 @@ func (h *Hub) parseIndex() error {
 		return fmt.Errorf("unable to read index file: %w", err)
 	}
 
-	if err := json.Unmarshal(bidx, &h.Items); err != nil {
+	if err := json.Unmarshal(bidx, &h.items); err != nil {
 		return fmt.Errorf("failed to unmarshal index: %w", err)
 	}
 
@@ -73,9 +74,9 @@ func (h *Hub) parseIndex() error {
 
 	// Iterate over the different types to complete the struct
 	for _, itemType := range ItemTypes {
-		log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
+		log.Tracef("%s: %d items", itemType, len(h.GetItemMap(itemType)))
 
-		for name, item := range h.Items[itemType] {
+		for name, item := range h.GetItemMap(itemType) {
 			item.hub = h
 			item.Name = name
 
@@ -101,13 +102,13 @@ func (h *Hub) ItemStats() []string {
 	tainted := 0
 
 	for _, itemType := range ItemTypes {
-		if len(h.Items[itemType]) == 0 {
+		if len(h.GetItemMap(itemType)) == 0 {
 			continue
 		}
 
-		loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
+		loaded += fmt.Sprintf("%d %s, ", len(h.GetItemMap(itemType)), itemType)
 
-		for _, item := range h.Items[itemType] {
+		for _, item := range h.GetItemMap(itemType) {
 			if item.IsLocal() {
 				local++
 			}
@@ -160,9 +161,17 @@ func (h *Hub) updateIndex() error {
 	return nil
 }
 
+func (h *Hub) addItem(item *Item) {
+	if h.items[item.Type] == nil {
+		h.items[item.Type] = make(map[string]*Item)
+	}
+
+	h.items[item.Type][item.Name] = item
+}
+
 // GetItemMap returns the map of items for a given type.
 func (h *Hub) GetItemMap(itemType string) map[string]*Item {
-	return h.Items[itemType]
+	return h.items[itemType]
 }
 
 // GetItem returns an item from hub based on its type and full name (author/name).
@@ -188,11 +197,12 @@ func (h *Hub) GetItemNames(itemType string) []string {
 
 // GetAllItems returns a slice of all the items of a given type, installed or not.
 func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
-	items, ok := h.Items[itemType]
+	if !slices.Contains(ItemTypes, itemType) {
-	if !ok {
+		return nil, fmt.Errorf("invalid item type %s", itemType)
-		return nil, fmt.Errorf("no %s in the hub index", itemType)
 	}
 
+	items := h.items[itemType]
+
 	ret := make([]*Item, len(items))
 
 	idx := 0
@@ -207,11 +217,12 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
 
 // GetInstalledItems returns a slice of the installed items of a given type.
 func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
-	items, ok := h.Items[itemType]
+	if !slices.Contains(ItemTypes, itemType) {
-	if !ok {
+		return nil, fmt.Errorf("invalid item type %s", itemType)
-		return nil, fmt.Errorf("no %s in the hub index", itemType)
 	}
 
+	items := h.items[itemType]
+
 	retItems := make([]*Item, 0)
 
 	for _, item := range items {

pkg/cwhub/item_test.go

@@ -63,7 +63,7 @@ func TestGetters(t *testing.T) {
 
 		// Add item and get it
 		item.Name += "nope"
-		hub.Items[item.Type][item.Name] = item
+		hub.addItem(item)
 
 		newitem := hub.GetItem(COLLECTIONS, item.Name)
 		require.NotNil(t, newitem)

pkg/cwhub/iteminstall_test.go

@@ -16,9 +16,9 @@ func testInstall(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
+	assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name)
+	assert.False(t, item.State.Installed, "%s should not be installed", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name)
 
 	err = item.enable()
 	require.NoError(t, err, "failed to enable %s", item.Name)
@@ -26,11 +26,11 @@ func testInstall(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
+	assert.True(t, item.State.Installed, "%s should be installed", item.Name)
 }
 
 func testTaint(hub *Hub, t *testing.T, item *Item) {
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name)
 
 	// truncate the file
 	f, err := os.Create(item.State.LocalPath)
@@ -41,11 +41,11 @@ func testTaint(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name)
+	assert.True(t, item.State.Tainted, "%s should be tainted", item.Name)
 }
 
 func testUpdate(hub *Hub, t *testing.T, item *Item) {
-	assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name)
+	assert.False(t, item.State.UpToDate, "%s should not be up-to-date", item.Name)
 
 	// Update it + check status
 	_, err := item.downloadLatest(true, true)
@@ -55,12 +55,12 @@ func testUpdate(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
+	assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name)
 }
 
 func testDisable(hub *Hub, t *testing.T, item *Item) {
-	assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
+	assert.True(t, item.State.Installed, "%s should be installed", item.Name)
 
 	// Remove
 	_, err := item.disable(false, false)
@@ -71,9 +71,9 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, hub.Warnings)
 
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name)
-	assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name)
+	assert.True(t, item.State.Downloaded, "%s should still be downloaded", item.Name)
 
 	// Purge
 	_, err = item.disable(true, false)
@@ -84,8 +84,8 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, hub.Warnings)
 
-	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name)
+	assert.False(t, item.State.Downloaded, "%s should not be downloaded", item.Name)
 }
 
 func TestInstallParser(t *testing.T) {
@@ -101,15 +101,11 @@ func TestInstallParser(t *testing.T) {
 	hub := envSetup(t)
 
 	// map iteration is random by itself
-	for _, it := range hub.Items[PARSERS] {
+	for _, it := range hub.GetItemMap(PARSERS) {
 		testInstall(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
 		testTaint(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
 		testUpdate(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
 		testDisable(hub, t, it)
-
 		break
 	}
 }
@@ -127,15 +123,11 @@ func TestInstallCollection(t *testing.T) {
 	hub := envSetup(t)
 
 	// map iteration is random by itself
-	for _, it := range hub.Items[COLLECTIONS] {
+	for _, it := range hub.GetItemMap(COLLECTIONS) {
 		testInstall(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
 		testTaint(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
 		testUpdate(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
 		testDisable(hub, t, it)
-
 		break
 	}
 }

pkg/cwhub/itemupgrade_test.go

@@ -13,20 +13,21 @@ import (
 func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 	hub := envSetup(t)
 
-	// fresh install of collection
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-
 	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
+	// fresh install of collection
+	require.False(t, item.State.Downloaded)
+	require.False(t, item.State.Installed)
+
 	require.NoError(t, item.Install(false, false))
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, item.State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, item.State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.True(t, item.State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.False(t, item.State.Tainted)
 
 	// This is the scenario that gets added in next version of collection
-	require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
+	require.Nil(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario"))
 
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
@@ -44,19 +45,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 
 	hub = getHubOrFail(t, hub.local, remote)
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
-
 	item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
+	require.True(t, item.State.Downloaded)
+	require.True(t, item.State.Installed)
+	require.False(t, item.State.UpToDate)
+	require.False(t, item.State.Tainted)
+
 	didUpdate, err := item.Upgrade(false)
 	require.NoError(t, err)
 	require.True(t, didUpdate)
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Downloaded)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed)
 }
 
 // Install a collection, disable a scenario.
@@ -64,19 +66,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	hub := envSetup(t)
 
-	// fresh install of collection
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
-
 	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
+	// fresh install of collection
+	require.False(t, item.State.Downloaded)
+	require.False(t, item.State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
+
 	require.NoError(t, item.Install(false, false))
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, item.State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, item.State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.True(t, item.State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.False(t, item.State.Tainted)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
 	item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
@@ -92,11 +95,12 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 
 	hub = getHubOrFail(t, hub.local, remote)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate)
 
 	hub, err = NewHub(hub.local, remote, true)
 	require.NoError(t, err, "failed to download index: %s", err)
@@ -107,7 +111,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	require.False(t, didUpdate)
 
 	hub = getHubOrFail(t, hub.local, remote)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 }
 
 // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
@@ -124,19 +128,20 @@ func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCf
 func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
 	hub := envSetup(t)
 
-	// fresh install of collection
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
-
 	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
+	// fresh install of collection
+	require.False(t, item.State.Downloaded)
+	require.False(t, item.State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
+
 	require.NoError(t, item.Install(false, false))
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, item.State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, item.State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.True(t, item.State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.False(t, item.State.Tainted)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
 	item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
@@ -152,12 +157,12 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
 
 	hub = getHubOrFail(t, hub.local, remote)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Downloaded) // this fails
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate)
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
@@ -167,7 +172,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
 	hub, err = NewHub(hub.local, remote, true)
 	require.NoError(t, err, "failed to download index: %s", err)
 
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 	hub = getHubOrFail(t, hub.local, remote)
 
 	item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
@@ -176,14 +181,14 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
 	require.True(t, didUpdate)
 
 	hub = getHubOrFail(t, hub.local, remote)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed)
 }
 
 func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
 	t.Helper()
 
-	c := hub.Items[COLLECTIONS][collection]
+	c := hub.GetItem(COLLECTIONS, collection)
 	require.NoError(t, c.checkSubItemVersions())
 }
 

pkg/cwhub/sync.go

@@ -226,7 +226,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 			if err != nil {
 				return err
 			}
-			h.Items[info.ftype][item.Name] = item
+			h.addItem(item)
 
 			return nil
 		}
@@ -246,7 +246,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 	// try to find which configuration item it is
 	log.Tracef("check [%s] of %s", info.fname, info.ftype)
 
-	for name, item := range h.Items[info.ftype] {
+	for _, item := range h.GetItemMap(info.ftype) {
 		if info.fname != item.FileName {
 			continue
 		}
@@ -287,8 +287,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 			return err
 		}
 
-		h.Items[info.ftype][name] = item
-
 		return nil
 	}
 
@@ -395,7 +393,7 @@ func (h *Hub) localSync() error {
 
 	warnings := make([]string, 0)
 
-	for _, item := range h.Items[COLLECTIONS] {
+	for _, item := range h.GetItemMap(COLLECTIONS) {
 		// check for cyclic dependencies
 		subs, err := item.descendants()
 		if err != nil {

pkg/hubtest/coverage.go

@@ -19,12 +19,12 @@ type Coverage struct {
 }
 
 func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
-	if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
+	if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 {
 		return nil, fmt.Errorf("no parsers in hub index")
 	}
 
 	// populate from hub, iterate in alphabetical order
-	pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS])
+	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.PARSERS))
 	coverage := make([]Coverage, len(pkeys))
 
 	for i, name := range pkeys {
@@ -105,12 +105,12 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
 }
 
 func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
-	if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
+	if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0  {
 		return nil, fmt.Errorf("no scenarios in hub index")
 	}
 
 	// populate from hub, iterate in alphabetical order
-	pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS])
+	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS))
 	coverage := make([]Coverage, len(pkeys))
 
 	for i, name := range pkeys {

pkg/hubtest/hubtest_item.go

@@ -146,7 +146,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 
-		if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
+		if hubParser := t.HubIndex.GetItem(cwhub.PARSERS, parser); hubParser != nil {
 			parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
@@ -232,7 +232,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 
-		if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
+		if hubScenario := t.HubIndex.GetItem(cwhub.SCENARIOS, scenario); hubScenario != nil {
 			scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
@@ -303,7 +303,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 
-		if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
+		if hubPostOverflow := t.HubIndex.GetItem(cwhub.POSTOVERFLOWS, postoverflow); hubPostOverflow != nil {
 			postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)

test/bats/20_hub.bats

@@ -126,3 +126,16 @@ teardown() {
     rune -0 cscli hub upgrade
     assert_stderr --partial "not upgrading foo.yaml: local item"
 }
+
+@test "cscli hub types" {
+    rune -0 cscli hub types -o raw
+    assert_line "parsers"
+    assert_line "postoverflows"
+    assert_line "scenarios"
+    assert_line "collections"
+    rune -0 cscli hub types -o human
+    rune -0 yq -o json <(output)
+    assert_json '["parsers","postoverflows","scenarios","collections"]'
+    rune -0 cscli hub types -o json
+    assert_json '["parsers","postoverflows","scenarios","collections"]'
+}

test/bin/preload-hub-items

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck disable=SC1091
+. "${THIS_DIR}/../.environment.sh"
+
+# pre-download everything but don't install anything
+
+echo -n "Purging existing hub..."
+
+types=$("$CSCLI" hub types -o raw)
+
+for itemtype in $types; do
+    "$CSCLI" "${itemtype}" delete --all --error --purge --force
+done
+
+echo " done."
+
+echo -n "Pre-downloading Hub content..."
+
+for itemtype in $types; do
+    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name')
+    if [[ -n "${ALL_ITEMS}" ]]; then
+        #shellcheck disable=SC2086
+        "$CSCLI" "$itemtype" install \
+            $ALL_ITEMS \
+            --download-only \
+            --error
+    fi
+done
+
+# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
+# so we have to delete the links manually, and leave the downloaded files in place
+
+for itemtype in $types; do
+    "$CSCLI" "$itemtype" delete --all --error
+done
+
+echo " done."

test/lib/config/config-global

@@ -54,50 +54,6 @@ remove_init_data() {
 
 # we need a separate function for initializing config when testing package
 # because we want to test the configuration as well
-preload_hub_items() {
-    # pre-download everything but don't install anything
-    # each test can install what it needs
-
-    echo "Purging existing hub..."
-
-    "$CSCLI" parsers delete --all --error --purge --force
-    "$CSCLI" scenarios delete --all --error --purge --force
-    "$CSCLI" postoverflows delete --all --error --purge --force
-    "$CSCLI" collections delete --all --error --purge --force
-
-    echo "Pre-downloading hub content..."
-
-    #shellcheck disable=SC2046
-    "$CSCLI" collections install \
-        $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" parsers install \
-        $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" scenarios install \
-        $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" postoverflows install \
-        $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
-        --download-only \
-        --error
-
-    # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
-    # so we have to delete the links manually, and leave the downloaded files in place
-
-    "$CSCLI" parsers delete --all --error
-    "$CSCLI" scenarios delete --all --error
-    "$CSCLI" postoverflows delete --all --error
-}
 
 make_init_data() {
     ./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
@@ -108,7 +64,7 @@ make_init_data() {
     # when installed packages are always using sqlite, so no need to regenerate
     # local credz for sqlite
 
-    preload_hub_items
+    ./bin/preload-hub-items
 
     [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto
 
@@ -145,7 +101,6 @@ load_init_data() {
     ./instance-db restore "${LOCAL_INIT_DIR}/database"
 }
 
-
 # ---------------------------
 
 [[ $# -lt 1 ]] && about

test/lib/config/config-local

@@ -101,51 +101,6 @@ config_generate() {
     ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml"
 }
 
-preload_hub_items() {
-    # pre-download everything but don't install anything
-    # each test can install what it needs
-
-    echo "Purging existing hub..."
-
-    "$CSCLI" parsers delete --all --error --purge --force
-    "$CSCLI" scenarios delete --all --error --purge --force
-    "$CSCLI" postoverflows delete --all --error --purge --force
-    "$CSCLI" collections delete --all --error --purge --force
-
-    echo "Pre-downloading hub content..."
-
-    #shellcheck disable=SC2046
-    "$CSCLI" collections install \
-        $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" parsers install \
-        $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" scenarios install \
-        $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" postoverflows install \
-        $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
-        --download-only \
-        --error
-
-    # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
-    # so we have to delete the links manually, and leave the downloaded files in place
-
-    "$CSCLI" parsers delete --all --error
-    "$CSCLI" scenarios delete --all --error
-    "$CSCLI" postoverflows delete --all --error
-}
-
 make_init_data() {
     ./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
 
@@ -163,7 +118,7 @@ make_init_data() {
     "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
     "$CSCLI" --warning hub update
 
-    preload_hub_items
+    ./bin/preload-hub-items
 
     mkdir -p "$LOCAL_INIT_DIR"
 
@@ -197,7 +152,6 @@ load_init_data() {
     ./instance-db restore "${LOCAL_INIT_DIR}/database"
 }
 
-
 # ---------------------------
 
 [[ $# -lt 1 ]] && about

test/lib/setup_file.sh

@@ -243,8 +243,16 @@ export -f assert_stderr_line
 hub_purge_all() {
     local CONFIG_DIR
     CONFIG_DIR=$(dirname "$CONFIG_YAML")
-    rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/*
+    rm -rf \
-    rm -rf "$CONFIG_DIR"/hub/collections/* "$CONFIG_DIR"/hub/parsers/*/* "$CONFIG_DIR"/hub/scenarios/* "$CONFIG_DIR"/hub/postoverflows/*
+        "$CONFIG_DIR"/collections/* \
+        "$CONFIG_DIR"/parsers/*/* \
+        "$CONFIG_DIR"/scenarios/* \
+        "$CONFIG_DIR"/postoverflows/*
+    rm -rf \
+        "$CONFIG_DIR"/hub/collections/* \
+        "$CONFIG_DIR"/hub/parsers/*/* \
+        "$CONFIG_DIR"/hub/scenarios/* \
+        "$CONFIG_DIR"/hub/postoverflows/*
     local DATA_DIR
     DATA_DIR=$(config_get .config_paths.data_dir)
     # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),