Merge branch 'master' into postoverflow_reinject_meta

.github/workflows/go-tests.yml

@@ -140,12 +140,13 @@ jobs:
         go install github.com/kyoh86/richgo@v0.3.10
         set -o pipefail
         make build BUILD_STATIC=1
-        make go-acc | richgo testfilter
+        make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
     - name: Run tests again, dynamic
       run: |
         make clean build
-        make go-acc | richgo testfilter
+        set -o pipefail
+        make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
     - name: Upload unit coverage to Codecov
       uses: codecov/codecov-action@v3

Makefile

@@ -232,8 +232,7 @@ test: testenv goversion
 # run the tests with localstack and coverage
 .PHONY: go-acc
 go-acc: testenv goversion
-	go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
-		sed 's/ *coverage:.*of statements in.*//'
+	go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
 
 # mock AWS services
 .PHONY: localstack

cmd/crowdsec-cli/dashboard.go

@@ -1,3 +1,5 @@
+//go:build linux
+
 package main
 
 import (
@@ -9,6 +11,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"unicode"
 
 	"github.com/AlecAivazis/survey/v2"
@@ -136,6 +139,9 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 			if err != nil {
 				return err
 			}
+			if err = chownDatabase(dockerGroup.Gid); err != nil {
+				return err
+			}
 			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
 			if err != nil {
 				return err
@@ -366,45 +372,56 @@ func disclaimer(forceYes *bool) error {
 }
 
 func checkGroups(forceYes *bool) (*user.Group, error) {
-	groupExist := false
 	dockerGroup, err := user.LookupGroup(crowdsecGroup)
 	if err == nil {
-		groupExist = true
+		return dockerGroup, nil
 	}
-	if !groupExist {
-		if !*forceYes {
-			var answer bool
-			prompt := &survey.Confirm{
-				Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
-				Default: true,
-			}
-			if err := survey.AskOne(prompt, &answer); err != nil {
-				return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
-			}
-			if !answer {
-				return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
-			}
-		}
-		groupAddCmd, err := exec.LookPath("groupadd")
-		if err != nil {
-			return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
+	if !*forceYes {
+		var answer bool
+		prompt := &survey.Confirm{
+			Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
+			Default: true,
 		}
-
-		groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
-		if err := groupAdd.Run(); err != nil {
-			return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
+		if err := survey.AskOne(prompt, &answer); err != nil {
+			return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
 		}
-		dockerGroup, err = user.LookupGroup(crowdsecGroup)
-		if err != nil {
-			return dockerGroup, fmt.Errorf("unable to lookup '%s' group: %+v", dockerGroup, err)
+		if !answer {
+			return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
 		}
 	}
-	intID, err := strconv.Atoi(dockerGroup.Gid)
+	groupAddCmd, err := exec.LookPath("groupadd")
 	if err != nil {
-		return dockerGroup, fmt.Errorf("unable to convert group ID to int: %s", err)
+		return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
 	}
-	if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
-		return dockerGroup, fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+
+	groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
+	if err := groupAdd.Run(); err != nil {
+		return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
 	}
-	return dockerGroup, nil
+	return user.LookupGroup(crowdsecGroup)
+}
+
+func chownDatabase(gid string) error {
+	intID, err := strconv.Atoi(gid)
+	if err != nil {
+		return fmt.Errorf("unable to convert group ID to int: %s", err)
+	}
+	if stat, err := os.Stat(csConfig.DbConfig.DbPath); !os.IsNotExist(err) {
+		info := stat.Sys()
+		if err := os.Chown(csConfig.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+			return fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+		}
+	}
+	if csConfig.DbConfig.Type == "sqlite" && csConfig.DbConfig.UseWal != nil && *csConfig.DbConfig.UseWal {
+		for _, ext := range []string{"-wal", "-shm"} {
+			file := csConfig.DbConfig.DbPath + ext
+			if stat, err := os.Stat(file); !os.IsNotExist(err) {
+				info := stat.Sys()
+				if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+					return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
+				}
+			}
+		}
+	}
+	return nil
 }

cmd/crowdsec-cli/dashboard_unsupported.go

@@ -0,0 +1,22 @@
+//go:build !linux
+
+package main
+
+import (
+	"runtime"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+func NewDashboardCmd() *cobra.Command {
+	var cmdDashboard = &cobra.Command{
+		Use:               "dashboard",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
+		},
+	}
+
+	return cmdDashboard
+}

cmd/crowdsec-cli/hub.go

@@ -1,11 +1,13 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@@ -29,6 +31,7 @@ cscli hub upgrade`,
 	cmdHub.AddCommand(NewHubListCmd())
 	cmdHub.AddCommand(NewHubUpdateCmd())
 	cmdHub.AddCommand(NewHubUpgradeCmd())
+	cmdHub.AddCommand(NewHubTypesCmd())
 
 	return cmdHub
 }
@@ -172,3 +175,40 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 
 	return cmdHubUpgrade
 }
+
+func runHubTypes(cmd *cobra.Command, args []string) error {
+	switch csConfig.Cscli.Output {
+	case "human":
+		s, err := yaml.Marshal(cwhub.ItemTypes)
+		if err != nil {
+			return err
+		}
+		fmt.Print(string(s))
+	case "json":
+		jsonStr, err := json.Marshal(cwhub.ItemTypes)
+		if err != nil {
+			return err
+		}
+		fmt.Println(string(jsonStr))
+	case "raw":
+		for _, itemType := range cwhub.ItemTypes {
+			fmt.Println(itemType)
+		}
+	}
+	return nil
+}
+
+func NewHubTypesCmd() *cobra.Command {
+	cmdHubTypes := &cobra.Command{
+		Use:   "types",
+		Short: "List supported item types",
+		Long: `
+List the types of supported hub items.
+`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		RunE:              runHubTypes,
+	}
+
+	return cmdHubTypes
+}

cmd/crowdsec-cli/item_suggest.go

@@ -12,10 +12,10 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-const MaxDistance = 7
+// suggestNearestMessage returns a message with the most similar item name, if one is found
+func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
+	const maxDistance = 7
 
-// SuggestNearestMessage returns a message with the most similar item name, if one is found
-func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
 	score := 100
 	nearest := ""
 
@@ -29,7 +29,7 @@ func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
 
 	msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
 
-	if score < MaxDistance {
+	if score < maxDistance {
 		msg += fmt.Sprintf(", did you mean '%s'?", nearest)
 	}
 

cmd/crowdsec-cli/itemcommands.go

@@ -36,7 +36,7 @@ type hubItemType struct {
 
 var hubItemTypes = map[string]hubItemType{
 	"parsers": {
-		name:      "parsers",
+		name:      cwhub.PARSERS,
 		singular:  "parser",
 		oneOrMore: "parser(s)",
 		help: cmdHelp{
@@ -68,7 +68,7 @@ List only enabled parsers unless "-a" or names are specified.`,
 		},
 	},
 	"postoverflows": {
-		name:      "postoverflows",
+		name:      cwhub.POSTOVERFLOWS,
 		singular:  "postoverflow",
 		oneOrMore: "postoverflow(s)",
 		help: cmdHelp{
@@ -100,7 +100,7 @@ List only enabled postoverflows unless "-a" or names are specified.`,
 		},
 	},
 	"scenarios": {
-		name:      "scenarios",
+		name:      cwhub.SCENARIOS,
 		singular:  "scenario",
 		oneOrMore: "scenario(s)",
 		help: cmdHelp{
@@ -132,7 +132,7 @@ List only enabled scenarios unless "-a" or names are specified.`,
 		},
 	},
 	"collections": {
-		name:      "collections",
+		name:      cwhub.COLLECTIONS,
 		singular:  "collection",
 		oneOrMore: "collection(s)",
 		help: cmdHelp{
@@ -214,7 +214,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
 		for _, name := range args {
 			item := hub.GetItem(it.name, name)
 			if item == nil {
-				msg := SuggestNearestMessage(hub, it.name, name)
+				msg := suggestNearestMessage(hub, it.name, name)
 				if !ignoreError {
 					return fmt.Errorf(msg)
 				}
@@ -319,6 +319,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
 					return err
 				}
 				if didRemove {
+					log.Infof("Removed %s", item.Name)
 					removed++
 				}
 			}
@@ -361,6 +362,8 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
 				removed++
 			}
 		}
+
+		log.Infof("Removed %d %s", removed, it.name)
 		if removed > 0 {
 			log.Infof(ReloadMessage())
 		}

cmd/crowdsec-cli/utils.go

@@ -13,14 +13,12 @@ import (
 )
 
 func printHelp(cmd *cobra.Command) {
-	err := cmd.Help()
-	if err != nil {
+	if err := cmd.Help(); err != nil {
 		log.Fatalf("unable to print help(): %s", err)
 	}
 }
 
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
-
 	/*if a range is provided, change the scope*/
 	if *ipRange != "" {
 		_, _, err := net.ParseCIDR(*ipRange)
@@ -50,7 +48,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 }
 
 func getDBClient() (*database.Client, error) {
-	var err error
 	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
 		return nil, err
 	}

pkg/apiserver/apic.go

@@ -618,12 +618,23 @@ func (a *apic) PullTop(forcePull bool) error {
 	}
 
 	// update blocklists
-	if err := a.UpdateBlocklists(data.Links, add_counters); err != nil {
+	if err := a.UpdateBlocklists(data.Links, add_counters, forcePull); err != nil {
 		return fmt.Errorf("while updating blocklists: %w", err)
 	}
 	return nil
 }
 
+// we receive a link to a blocklist, we pull the content of the blocklist and we create one alert
+func (a *apic) PullBlocklist(blocklist *modelscapi.BlocklistLink, forcePull bool) error {
+	add_counters, _ := makeAddAndDeleteCounters()
+	if err := a.UpdateBlocklists(&modelscapi.GetDecisionsStreamResponseLinks{
+		Blocklists: []*modelscapi.BlocklistLink{blocklist},
+	}, add_counters, forcePull); err != nil {
+		return fmt.Errorf("while pulling blocklist: %w", err)
+	}
+	return nil
+}
+
 // if decisions is whitelisted: return representation of the whitelist ip or cidr
 // if not whitelisted: empty string
 func (a *apic) whitelistedBy(decision *models.Decision) string {
@@ -710,7 +721,7 @@ func (a *apic) ShouldForcePullBlocklist(blocklist *modelscapi.BlocklistLink) (bo
 	return false, nil
 }
 
-func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscapi.BlocklistLink, add_counters map[string]map[string]int) error {
+func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscapi.BlocklistLink, add_counters map[string]map[string]int, forcePull bool) error {
 	if blocklist.Scope == nil {
 		log.Warningf("blocklist has no scope")
 		return nil
@@ -719,12 +730,16 @@ func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscap
 		log.Warningf("blocklist has no duration")
 		return nil
 	}
-	forcePull, err := a.ShouldForcePullBlocklist(blocklist)
-	if err != nil {
-		return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
+	if !forcePull {
+		_forcePull, err := a.ShouldForcePullBlocklist(blocklist)
+		if err != nil {
+			return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
+		}
+		forcePull = _forcePull
 	}
 	blocklistConfigItemName := fmt.Sprintf("blocklist:%s:last_pull", *blocklist.Name)
 	var lastPullTimestamp *string
+	var err error
 	if !forcePull {
 		lastPullTimestamp, err = a.dbClient.GetConfigItem(blocklistConfigItemName)
 		if err != nil {
@@ -764,7 +779,7 @@ func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscap
 	return nil
 }
 
-func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLinks, add_counters map[string]map[string]int) error {
+func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLinks, add_counters map[string]map[string]int, forcePull bool) error {
 	if links == nil {
 		return nil
 	}
@@ -778,7 +793,7 @@ func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLink
 		return fmt.Errorf("while creating default client: %w", err)
 	}
 	for _, blocklist := range links.Blocklists {
-		if err := a.updateBlocklist(defaultClient, blocklist, add_counters); err != nil {
+		if err := a.updateBlocklist(defaultClient, blocklist, add_counters, forcePull); err != nil {
 			return err
 		}
 	}

pkg/apiserver/apic_metrics.go

@@ -26,8 +26,8 @@ func (a *apic) GetMetrics() (*models.Metrics, error) {
 		machinesInfo[i] = &models.MetricsAgentInfo{
 			Version:    machine.Version,
 			Name:       machine.MachineId,
-			LastUpdate: machine.UpdatedAt.String(),
-			LastPush:   ptr.OrEmpty(machine.LastPush).String(),
+			LastUpdate: machine.UpdatedAt.Format(time.RFC3339),
+			LastPush:   ptr.OrEmpty(machine.LastPush).Format(time.RFC3339),
 		}
 	}
 
@@ -43,7 +43,7 @@ func (a *apic) GetMetrics() (*models.Metrics, error) {
 			Version:    bouncer.Version,
 			CustomName: bouncer.Name,
 			Name:       bouncer.Type,
-			LastPull:   bouncer.LastPull.String(),
+			LastPull:   bouncer.LastPull.Format(time.RFC3339),
 		}
 	}
 
@@ -81,7 +81,7 @@ func (a *apic) SendMetrics(stop chan (bool)) {
 	const checkInt = 20 * time.Second
 
 	// intervals must always be > 0
-	metInts := []time.Duration{1*time.Millisecond, a.metricsIntervalFirst, a.metricsInterval}
+	metInts := []time.Duration{1 * time.Millisecond, a.metricsIntervalFirst, a.metricsInterval}
 
 	log.Infof("Start sending metrics to CrowdSec Central API (interval: %s once, then %s)",
 		metInts[1].Round(time.Second), metInts[2])
@@ -123,7 +123,7 @@ func (a *apic) SendMetrics(stop chan (bool)) {
 			reloadMachineIDs()
 			if !slices.Equal(oldIDs, machineIDs) {
 				log.Infof("capi metrics: machines changed, immediate send")
-				metTicker.Reset(1*time.Millisecond)
+				metTicker.Reset(1 * time.Millisecond)
 			}
 		case <-metTicker.C:
 			metTicker.Stop()

pkg/apiserver/apic_test.go

@@ -309,30 +309,30 @@ func TestAPICGetMetrics(t *testing.T) {
 				Bouncers: []*models.MetricsBouncerInfo{
 					{
 						CustomName: "1",
-						LastPull:   time.Time{}.String(),
+						LastPull:   time.Time{}.Format(time.RFC3339),
 					}, {
 						CustomName: "2",
-						LastPull:   time.Time{}.String(),
+						LastPull:   time.Time{}.Format(time.RFC3339),
 					}, {
 						CustomName: "3",
-						LastPull:   time.Time{}.String(),
+						LastPull:   time.Time{}.Format(time.RFC3339),
 					},
 				},
 				Machines: []*models.MetricsAgentInfo{
 					{
 						Name:       "a",
-						LastPush:   time.Time{}.String(),
-						LastUpdate: time.Time{}.String(),
+						LastPush:   time.Time{}.Format(time.RFC3339),
+						LastUpdate: time.Time{}.Format(time.RFC3339),
 					},
 					{
 						Name:       "b",
-						LastPush:   time.Time{}.String(),
-						LastUpdate: time.Time{}.String(),
+						LastPush:   time.Time{}.Format(time.RFC3339),
+						LastUpdate: time.Time{}.Format(time.RFC3339),
 					},
 					{
 						Name:       "c",
-						LastPush:   time.Time{}.String(),
-						LastUpdate: time.Time{}.String(),
+						LastPush:   time.Time{}.Format(time.RFC3339),
+						LastUpdate: time.Time{}.Format(time.RFC3339),
 					},
 				},
 			},
@@ -973,6 +973,37 @@ func TestAPICPullTopBLCacheForceCall(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestAPICPullBlocklistCall(t *testing.T) {
+	api := getAPIC(t)
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist1", func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, "", req.Header.Get("If-Modified-Since"))
+		return httpmock.NewStringResponse(200, "1.2.3.4"), nil
+	})
+	url, err := url.ParseRequestURI("http://api.crowdsec.net/")
+	require.NoError(t, err)
+
+	apic, err := apiclient.NewDefaultClient(
+		url,
+		"/api",
+		fmt.Sprintf("crowdsec/%s", version.String()),
+		nil,
+	)
+	require.NoError(t, err)
+
+	api.apiClient = apic
+	err = api.PullBlocklist(&modelscapi.BlocklistLink{
+		URL:         ptr.Of("http://api.crowdsec.net/blocklist1"),
+		Name:        ptr.Of("blocklist1"),
+		Scope:       ptr.Of("Ip"),
+		Remediation: ptr.Of("ban"),
+		Duration:    ptr.Of("24h"),
+	}, true)
+	require.NoError(t, err)
+}
+
 func TestAPICPush(t *testing.T) {
 	tests := []struct {
 		name          string

pkg/apiserver/papi_cmd.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/crowdsecurity/crowdsec/pkg/modelscapi"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
@@ -19,6 +20,23 @@ type deleteDecisions struct {
 	Decisions []string `json:"decisions"`
 }
 
+type blocklistLink struct {
+	// blocklist name
+	Name string `json:"name"`
+	// blocklist url
+	Url string `json:"url"`
+	// blocklist remediation
+	Remediation string `json:"remediation"`
+	// blocklist scope
+	Scope string `json:"scope,omitempty"`
+	// blocklist duration
+	Duration string `json:"duration,omitempty"`
+}
+
+type forcePull struct {
+	Blocklist *blocklistLink `json:"blocklist,omitempty"`
+}
+
 func DecisionCmd(message *Message, p *Papi, sync bool) error {
 	switch message.Header.OperationCmd {
 	case "delete":
@@ -144,11 +162,35 @@ func ManagementCmd(message *Message, p *Papi, sync bool) error {
 		log.Infof("Received reauth command from PAPI, resetting token")
 		p.apiClient.GetClient().Transport.(*apiclient.JWTTransport).ResetToken()
 	case "force_pull":
-		log.Infof("Received force_pull command from PAPI, pulling community and 3rd-party blocklists")
-		err := p.apic.PullTop(true)
+		data, err := json.Marshal(message.Data)
 		if err != nil {
-			return fmt.Errorf("failed to force pull operation: %s", err)
+			return err
+		}
+		forcePullMsg := forcePull{}
+		if err := json.Unmarshal(data, &forcePullMsg); err != nil {
+			return fmt.Errorf("message for '%s' contains bad data format: %s", message.Header.OperationType, err)
 		}
+
+		if forcePullMsg.Blocklist == nil {
+			log.Infof("Received force_pull command from PAPI, pulling community and 3rd-party blocklists")
+			err = p.apic.PullTop(true)
+			if err != nil {
+				return fmt.Errorf("failed to force pull operation: %s", err)
+			}
+		} else {
+			log.Infof("Received force_pull command from PAPI, pulling blocklist %s", forcePullMsg.Blocklist.Name)
+			err = p.apic.PullBlocklist(&modelscapi.BlocklistLink{
+				Name:        &forcePullMsg.Blocklist.Name,
+				URL:         &forcePullMsg.Blocklist.Url,
+				Remediation: &forcePullMsg.Blocklist.Remediation,
+				Scope:       &forcePullMsg.Blocklist.Scope,
+				Duration:    &forcePullMsg.Blocklist.Duration,
+			}, true)
+			if err != nil {
+				return fmt.Errorf("failed to force pull operation: %s", err)
+			}
+		}
+
 	default:
 		return fmt.Errorf("unknown command '%s' for operation type '%s'", message.Header.OperationCmd, message.Header.OperationType)
 	}

pkg/cwhub/cwhub.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 )
@@ -30,3 +31,10 @@ func safePath(dir, filePath string) (string, error) {
 
 	return absFilePath, nil
 }
+
+// SortItemSlice sorts a slice of items by name, case insensitive.
+func SortItemSlice(items []*Item) {
+	sort.Slice(items, func(i, j int) bool {
+		return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
+	})
+}

pkg/cwhub/dataset.go

@@ -6,9 +6,10 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"time"
 
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
@@ -51,6 +52,62 @@ func downloadFile(url string, destPath string) error {
 	return nil
 }
 
+// needsUpdate checks if a data file has to be downloaded (or updated).
+// if the local file doesn't exist, update.
+// if the remote is newer than the local file, update.
+// if the remote has no modification date, but local file has been modified > a week ago, update.
+func needsUpdate(destPath string, url string) bool {
+	fileInfo, err := os.Stat(destPath)
+	switch {
+	case os.IsNotExist(err):
+		return true
+	case err != nil:
+		log.Errorf("while getting %s: %s", destPath, err)
+		return true
+	}
+
+	resp, err := hubClient.Head(url)
+	if err != nil {
+		log.Errorf("while getting %s: %s", url, err)
+		// Head failed, Get would likely fail too -> no update
+		return false
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		log.Errorf("bad http code %d for %s", resp.StatusCode, url)
+		return false
+	}
+
+	// update if local file is older than this
+	shelfLife := 7 * 24 * time.Hour
+
+	lastModify := fileInfo.ModTime()
+
+	localIsOld := lastModify.Add(shelfLife).Before(time.Now())
+
+	remoteLastModified := resp.Header.Get("Last-Modified")
+	if remoteLastModified == "" {
+		if localIsOld {
+			log.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife)
+		}
+		return localIsOld
+	}
+
+	lastAvailable, err := time.Parse(time.RFC1123, remoteLastModified)
+	if err != nil {
+		log.Warningf("while parsing last modified date for %s: %s", url, err)
+		return localIsOld
+	}
+
+	if lastModify.Before(lastAvailable) {
+		log.Infof("new version available, updating %s", destPath)
+		return true
+	}
+
+	return false
+}
+
 // downloadDataSet downloads all the data files for an item.
 func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
 	dec := yaml.NewDecoder(reader)
@@ -72,9 +129,7 @@ func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
 				return err
 			}
 
-			if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
-				log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
-
+			if force || needsUpdate(destPath, dataS.SourceURL) {
 				if err := downloadFile(dataS.SourceURL, destPath); err != nil {
 					return fmt.Errorf("while getting data: %w", err)
 				}

pkg/cwhub/enable.go

@@ -1,190 +0,0 @@
-package cwhub
-
-// Enable/disable items already downloaded
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
-// (eg. /etc/crowdsec/collections/xyz.yaml).
-// Raises an error if the path goes outside of the install dir.
-func (i *Item) installPath() (string, error) {
-	p := i.Type
-	if i.Stage != "" {
-		p = filepath.Join(p, i.Stage)
-	}
-
-	return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
-}
-
-// downloadPath returns the location of the actual config file in the hub
-// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
-// Raises an error if the path goes outside of the hub dir.
-func (i *Item) downloadPath() (string, error) {
-	ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
-	if err != nil {
-		return "", err
-	}
-
-	return ret, nil
-}
-
-// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
-func (i *Item) createInstallLink() error {
-	dest, err := i.installPath()
-	if err != nil {
-		return err
-	}
-
-	destDir := filepath.Dir(dest)
-	if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
-		return fmt.Errorf("while creating %s: %w", destDir, err)
-	}
-
-	if _, err = os.Lstat(dest); !os.IsNotExist(err) {
-		log.Infof("%s already exists.", dest)
-		return nil
-	}
-
-	src, err := i.downloadPath()
-	if err != nil {
-		return err
-	}
-
-	if err = os.Symlink(src, dest); err != nil {
-		return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
-	}
-
-	return nil
-}
-
-// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
-func (i *Item) enable() error {
-	if i.State.Installed {
-		if i.State.Tainted {
-			return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
-		}
-
-		if i.IsLocal() {
-			return fmt.Errorf("%s is local, won't enable", i.Name)
-		}
-
-		// if it's a collection, check sub-items even if the collection file itself is up-to-date
-		if i.State.UpToDate && !i.HasSubItems() {
-			log.Tracef("%s is installed and up-to-date, skip.", i.Name)
-			return nil
-		}
-	}
-
-	for _, sub := range i.SubItems() {
-		if err := sub.enable(); err != nil {
-			return fmt.Errorf("while installing %s: %w", sub.Name, err)
-		}
-	}
-
-	if err := i.createInstallLink(); err != nil {
-		return err
-	}
-
-	log.Infof("Enabled %s: %s", i.Type, i.Name)
-	i.State.Installed = true
-
-	return nil
-}
-
-// purge removes the actual config file that was downloaded.
-func (i *Item) purge() error {
-	if !i.State.Downloaded {
-		log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
-		return nil
-	}
-
-	src, err := i.downloadPath()
-	if err != nil {
-		return err
-	}
-
-	if err := os.Remove(src); err != nil {
-		if os.IsNotExist(err) {
-			log.Debugf("%s doesn't exist, no need to remove", src)
-			return nil
-		}
-
-		return fmt.Errorf("while removing file: %w", err)
-	}
-
-	i.State.Downloaded = false
-	log.Infof("Removed source file [%s]: %s", i.Name, src)
-
-	return nil
-}
-
-// removeInstallLink removes the symlink to the downloaded content.
-func (i *Item) removeInstallLink() error {
-	syml, err := i.installPath()
-	if err != nil {
-		return err
-	}
-
-	stat, err := os.Lstat(syml)
-	if err != nil {
-		return err
-	}
-
-	// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
-	if stat.Mode()&os.ModeSymlink == 0 {
-		log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
-		return fmt.Errorf("%s isn't managed by hub", i.Name)
-	}
-
-	hubpath, err := os.Readlink(syml)
-	if err != nil {
-		return fmt.Errorf("while reading symlink: %w", err)
-	}
-
-	src, err := i.downloadPath()
-	if err != nil {
-		return err
-	}
-
-	if hubpath != src {
-		log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
-		return fmt.Errorf("%s isn't managed by hub", i.Name)
-	}
-
-	if err := os.Remove(syml); err != nil {
-		return fmt.Errorf("while removing symlink: %w", err)
-	}
-
-	log.Infof("Removed symlink [%s]: %s", i.Name, syml)
-
-	return nil
-}
-
-// disable removes the install link, and optionally the downloaded content.
-func (i *Item) disable(purge bool, force bool) error {
-	err := i.removeInstallLink()
-	if os.IsNotExist(err) {
-		if !purge && !force {
-			link, _ := i.installPath()
-			return fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
-		}
-	} else if err != nil {
-		return err
-	}
-
-	i.State.Installed = false
-
-	if purge {
-		if err := i.purge(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

pkg/cwhub/hub.go

@@ -9,13 +9,14 @@ import (
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+	"slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 )
 
 // Hub is the main structure for the package.
 type Hub struct {
-	Items    HubItems // Items read from HubDir and InstallDir
+	items    HubItems // Items read from HubDir and InstallDir
 	local    *csconfig.LocalHubCfg
 	remote   *RemoteHubCfg
 	Warnings []string // Warnings encountered during sync
@@ -65,7 +66,7 @@ func (h *Hub) parseIndex() error {
 		return fmt.Errorf("unable to read index file: %w", err)
 	}
 
-	if err := json.Unmarshal(bidx, &h.Items); err != nil {
+	if err := json.Unmarshal(bidx, &h.items); err != nil {
 		return fmt.Errorf("failed to unmarshal index: %w", err)
 	}
 
@@ -73,9 +74,9 @@ func (h *Hub) parseIndex() error {
 
 	// Iterate over the different types to complete the struct
 	for _, itemType := range ItemTypes {
-		log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
+		log.Tracef("%s: %d items", itemType, len(h.GetItemMap(itemType)))
 
-		for name, item := range h.Items[itemType] {
+		for name, item := range h.GetItemMap(itemType) {
 			item.hub = h
 			item.Name = name
 
@@ -101,13 +102,13 @@ func (h *Hub) ItemStats() []string {
 	tainted := 0
 
 	for _, itemType := range ItemTypes {
-		if len(h.Items[itemType]) == 0 {
+		if len(h.GetItemMap(itemType)) == 0 {
 			continue
 		}
 
-		loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
+		loaded += fmt.Sprintf("%d %s, ", len(h.GetItemMap(itemType)), itemType)
 
-		for _, item := range h.Items[itemType] {
+		for _, item := range h.GetItemMap(itemType) {
 			if item.IsLocal() {
 				local++
 			}
@@ -159,3 +160,92 @@ func (h *Hub) updateIndex() error {
 
 	return nil
 }
+
+func (h *Hub) addItem(item *Item) {
+	if h.items[item.Type] == nil {
+		h.items[item.Type] = make(map[string]*Item)
+	}
+
+	h.items[item.Type][item.Name] = item
+}
+
+// GetItemMap returns the map of items for a given type.
+func (h *Hub) GetItemMap(itemType string) map[string]*Item {
+	return h.items[itemType]
+}
+
+// GetItem returns an item from hub based on its type and full name (author/name).
+func (h *Hub) GetItem(itemType string, itemName string) *Item {
+	return h.GetItemMap(itemType)[itemName]
+}
+
+// GetItemNames returns a slice of (full) item names for a given type
+// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
+func (h *Hub) GetItemNames(itemType string) []string {
+	m := h.GetItemMap(itemType)
+	if m == nil {
+		return nil
+	}
+
+	names := make([]string, 0, len(m))
+	for k := range m {
+		names = append(names, k)
+	}
+
+	return names
+}
+
+// GetAllItems returns a slice of all the items of a given type, installed or not.
+func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
+	if !slices.Contains(ItemTypes, itemType) {
+		return nil, fmt.Errorf("invalid item type %s", itemType)
+	}
+
+	items := h.items[itemType]
+
+	ret := make([]*Item, len(items))
+
+	idx := 0
+
+	for _, item := range items {
+		ret[idx] = item
+		idx++
+	}
+
+	return ret, nil
+}
+
+// GetInstalledItems returns a slice of the installed items of a given type.
+func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
+	if !slices.Contains(ItemTypes, itemType) {
+		return nil, fmt.Errorf("invalid item type %s", itemType)
+	}
+
+	items := h.items[itemType]
+
+	retItems := make([]*Item, 0)
+
+	for _, item := range items {
+		if item.State.Installed {
+			retItems = append(retItems, item)
+		}
+	}
+
+	return retItems, nil
+}
+
+// GetInstalledItemNames returns the names of the installed items of a given type.
+func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
+	items, err := h.GetInstalledItems(itemType)
+	if err != nil {
+		return nil, err
+	}
+
+	retStr := make([]string, len(items))
+
+	for idx, it := range items {
+		retStr[idx] = it.Name
+	}
+
+	return retStr, nil
+}

pkg/cwhub/items.go → pkg/cwhub/item.go

@@ -3,8 +3,7 @@ package cwhub
 import (
 	"encoding/json"
 	"fmt"
-	"sort"
-	"strings"
+	"path/filepath"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/enescakir/emoji"
@@ -68,9 +67,9 @@ type Item struct {
 	Author      string   `json:"author,omitempty" yaml:"author,omitempty"`
 	References  []string `json:"references,omitempty" yaml:"references,omitempty"`
 
-	RemotePath string                 `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
-	Version    string                 `json:"version,omitempty" yaml:"version,omitempty"`  // the last available version
-	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`                // all the known versions
+	RemotePath string                 `json:"path,omitempty" yaml:"path,omitempty"`       // path relative to the base URL eg. /parsers/stage/author/file.yaml
+	Version    string                 `json:"version,omitempty" yaml:"version,omitempty"` // the last available version
+	Versions   map[string]ItemVersion `json:"versions,omitempty"  yaml:"-"`               // all the known versions
 
 	// if it's a collection, it can have sub items
 	Parsers       []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
@@ -79,6 +78,30 @@ type Item struct {
 	Collections   []string `json:"collections,omitempty" yaml:"collections,omitempty"`
 }
 
+// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
+// (eg. /etc/crowdsec/collections/xyz.yaml).
+// Raises an error if the path goes outside of the install dir.
+func (i *Item) installPath() (string, error) {
+	p := i.Type
+	if i.Stage != "" {
+		p = filepath.Join(p, i.Stage)
+	}
+
+	return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
+}
+
+// downloadPath returns the location of the actual config file in the hub
+// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
+// Raises an error if the path goes outside of the hub dir.
+func (i *Item) downloadPath() (string, error) {
+	ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
+	if err != nil {
+		return "", err
+	}
+
+	return ret, nil
+}
+
 // HasSubItems returns true if items of this type can have sub-items. Currently only collections.
 func (i *Item) HasSubItems() bool {
 	return i.Type == COLLECTIONS
@@ -225,6 +248,48 @@ func (i *Item) Ancestors() []*Item {
 	return ret
 }
 
+// descendants returns a list of all (direct or indirect) dependencies of the item.
+func (i *Item) descendants() ([]*Item, error) {
+	var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
+
+	collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
+		if item == nil {
+			return nil
+		}
+
+		if visited[item] {
+			return nil
+		}
+
+		visited[item] = true
+
+		for _, subItem := range item.SubItems() {
+			if subItem == i {
+				return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
+			}
+
+			*result = append(*result, subItem)
+
+			err := collectSubItems(subItem, visited, result)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	ret := []*Item{}
+	visited := map[*Item]bool{}
+
+	err := collectSubItems(i, visited, &ret)
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
 // InstallStatus returns the status of the item as a string and an emoji
 // (eg. "enabled,update-available" and emoji.Warning).
 func (i *Item) InstallStatus() (string, emoji.Emoji) {
@@ -295,89 +360,3 @@ func (i *Item) versionStatus() int {
 func (i *Item) validPath(dirName, fileName string) bool {
 	return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
 }
-
-// GetItemMap returns the map of items for a given type.
-func (h *Hub) GetItemMap(itemType string) map[string]*Item {
-	return h.Items[itemType]
-}
-
-// GetItem returns an item from hub based on its type and full name (author/name).
-func (h *Hub) GetItem(itemType string, itemName string) *Item {
-	return h.GetItemMap(itemType)[itemName]
-}
-
-// GetItemNames returns a slice of (full) item names for a given type
-// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
-func (h *Hub) GetItemNames(itemType string) []string {
-	m := h.GetItemMap(itemType)
-	if m == nil {
-		return nil
-	}
-
-	names := make([]string, 0, len(m))
-	for k := range m {
-		names = append(names, k)
-	}
-
-	return names
-}
-
-// GetAllItems returns a slice of all the items of a given type, installed or not.
-func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
-	items, ok := h.Items[itemType]
-	if !ok {
-		return nil, fmt.Errorf("no %s in the hub index", itemType)
-	}
-
-	ret := make([]*Item, len(items))
-
-	idx := 0
-
-	for _, item := range items {
-		ret[idx] = item
-		idx++
-	}
-
-	return ret, nil
-}
-
-// GetInstalledItems returns a slice of the installed items of a given type.
-func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
-	items, ok := h.Items[itemType]
-	if !ok {
-		return nil, fmt.Errorf("no %s in the hub index", itemType)
-	}
-
-	retItems := make([]*Item, 0)
-
-	for _, item := range items {
-		if item.State.Installed {
-			retItems = append(retItems, item)
-		}
-	}
-
-	return retItems, nil
-}
-
-// GetInstalledItemNames returns the names of the installed items of a given type.
-func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
-	items, err := h.GetInstalledItems(itemType)
-	if err != nil {
-		return nil, err
-	}
-
-	retStr := make([]string, len(items))
-
-	for idx, it := range items {
-		retStr[idx] = it.Name
-	}
-
-	return retStr, nil
-}
-
-// SortItemSlice sorts a slice of items by name, case insensitive.
-func SortItemSlice(items []*Item) {
-	sort.Slice(items, func(i, j int) bool {
-		return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
-	})
-}

pkg/cwhub/items_test.go → pkg/cwhub/item_test.go

@@ -63,7 +63,7 @@ func TestGetters(t *testing.T) {
 
 		// Add item and get it
 		item.Name += "nope"
-		hub.Items[item.Type][item.Name] = item
+		hub.addItem(item)
 
 		newitem := hub.GetItem(COLLECTIONS, item.Name)
 		require.NotNil(t, newitem)

pkg/cwhub/iteminstall.go

@@ -0,0 +1,70 @@
+package cwhub
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
+func (i *Item) enable() error {
+	if i.State.Installed {
+		if i.State.Tainted {
+			return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
+		}
+
+		if i.IsLocal() {
+			return fmt.Errorf("%s is local, won't enable", i.Name)
+		}
+
+		// if it's a collection, check sub-items even if the collection file itself is up-to-date
+		if i.State.UpToDate && !i.HasSubItems() {
+			log.Tracef("%s is installed and up-to-date, skip.", i.Name)
+			return nil
+		}
+	}
+
+	for _, sub := range i.SubItems() {
+		if err := sub.enable(); err != nil {
+			return fmt.Errorf("while installing %s: %w", sub.Name, err)
+		}
+	}
+
+	if err := i.createInstallLink(); err != nil {
+		return err
+	}
+
+	log.Infof("Enabled %s: %s", i.Type, i.Name)
+	i.State.Installed = true
+
+	return nil
+}
+
+// Install installs the item from the hub, downloading it if needed.
+func (i *Item) Install(force bool, downloadOnly bool) error {
+	if downloadOnly && i.State.Downloaded && i.State.UpToDate {
+		log.Infof("%s is already downloaded and up-to-date", i.Name)
+
+		if !force {
+			return nil
+		}
+	}
+
+	filePath, err := i.downloadLatest(force, true)
+	if err != nil {
+		return fmt.Errorf("while downloading %s: %w", i.Name, err)
+	}
+
+	if downloadOnly {
+		log.Infof("Downloaded %s to %s", i.Name, filePath)
+		return nil
+	}
+
+	if err := i.enable(); err != nil {
+		return fmt.Errorf("while enabling %s: %w", i.Name, err)
+	}
+
+	log.Infof("Enabled %s", i.Name)
+
+	return nil
+}

pkg/cwhub/enable_test.go → pkg/cwhub/iteminstall_test.go

@@ -16,9 +16,9 @@ func testInstall(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
+	assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, item.State.Installed, "%s should not be installed", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name)
 
 	err = item.enable()
 	require.NoError(t, err, "failed to enable %s", item.Name)
@@ -26,11 +26,11 @@ func testInstall(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
+	assert.True(t, item.State.Installed, "%s should be installed", item.Name)
 }
 
 func testTaint(hub *Hub, t *testing.T, item *Item) {
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name)
 
 	// truncate the file
 	f, err := os.Create(item.State.LocalPath)
@@ -41,11 +41,11 @@ func testTaint(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name)
+	assert.True(t, item.State.Tainted, "%s should be tainted", item.Name)
 }
 
 func testUpdate(hub *Hub, t *testing.T, item *Item) {
-	assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name)
+	assert.False(t, item.State.UpToDate, "%s should not be up-to-date", item.Name)
 
 	// Update it + check status
 	_, err := item.downloadLatest(true, true)
@@ -55,15 +55,15 @@ func testUpdate(hub *Hub, t *testing.T, item *Item) {
 	err = hub.localSync()
 	require.NoError(t, err, "failed to run localSync")
 
-	assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
+	assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name)
 }
 
 func testDisable(hub *Hub, t *testing.T, item *Item) {
-	assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
+	assert.True(t, item.State.Installed, "%s should be installed", item.Name)
 
 	// Remove
-	err := item.disable(false, false)
+	_, err := item.disable(false, false)
 	require.NoError(t, err, "failed to disable %s", item.Name)
 
 	// Local sync and check status
@@ -71,12 +71,12 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, hub.Warnings)
 
-	assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
-	assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name)
+	assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name)
+	assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name)
+	assert.True(t, item.State.Downloaded, "%s should still be downloaded", item.Name)
 
 	// Purge
-	err = item.disable(true, false)
+	_, err = item.disable(true, false)
 	require.NoError(t, err, "failed to purge %s", item.Name)
 
 	// Local sync and check status
@@ -84,8 +84,8 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
 	require.NoError(t, err, "failed to run localSync")
 	require.Empty(t, hub.Warnings)
 
-	assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
-	assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name)
+	assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name)
+	assert.False(t, item.State.Downloaded, "%s should not be downloaded", item.Name)
 }
 
 func TestInstallParser(t *testing.T) {
@@ -101,15 +101,11 @@ func TestInstallParser(t *testing.T) {
 	hub := envSetup(t)
 
 	// map iteration is random by itself
-	for _, it := range hub.Items[PARSERS] {
+	for _, it := range hub.GetItemMap(PARSERS) {
 		testInstall(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
 		testTaint(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
 		testUpdate(hub, t, it)
-		it = hub.Items[PARSERS][it.Name]
 		testDisable(hub, t, it)
-
 		break
 	}
 }
@@ -127,15 +123,11 @@ func TestInstallCollection(t *testing.T) {
 	hub := envSetup(t)
 
 	// map iteration is random by itself
-	for _, it := range hub.Items[COLLECTIONS] {
+	for _, it := range hub.GetItemMap(COLLECTIONS) {
 		testInstall(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
 		testTaint(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
 		testUpdate(hub, t, it)
-		it = hub.Items[COLLECTIONS][it.Name]
 		testDisable(hub, t, it)
-
 		break
 	}
 }

pkg/cwhub/itemlink.go

@@ -0,0 +1,80 @@
+package cwhub
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// createInstallLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
+func (i *Item) createInstallLink() error {
+	dest, err := i.installPath()
+	if err != nil {
+		return err
+	}
+
+	destDir := filepath.Dir(dest)
+	if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
+		return fmt.Errorf("while creating %s: %w", destDir, err)
+	}
+
+	if _, err = os.Lstat(dest); !os.IsNotExist(err) {
+		log.Infof("%s already exists.", dest)
+		return nil
+	}
+
+	src, err := i.downloadPath()
+	if err != nil {
+		return err
+	}
+
+	if err = os.Symlink(src, dest); err != nil {
+		return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
+	}
+
+	return nil
+}
+
+// removeInstallLink removes the symlink to the downloaded content.
+func (i *Item) removeInstallLink() error {
+	syml, err := i.installPath()
+	if err != nil {
+		return err
+	}
+
+	stat, err := os.Lstat(syml)
+	if err != nil {
+		return err
+	}
+
+	// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
+	if stat.Mode()&os.ModeSymlink == 0 {
+		log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
+		return fmt.Errorf("%s isn't managed by hub", i.Name)
+	}
+
+	hubpath, err := os.Readlink(syml)
+	if err != nil {
+		return fmt.Errorf("while reading symlink: %w", err)
+	}
+
+	src, err := i.downloadPath()
+	if err != nil {
+		return err
+	}
+
+	if hubpath != src {
+		log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
+		return fmt.Errorf("%s isn't managed by hub", i.Name)
+	}
+
+	if err := os.Remove(syml); err != nil {
+		return fmt.Errorf("while removing symlink: %w", err)
+	}
+
+	log.Infof("Removed symlink [%s]: %s", i.Name, syml)
+
+	return nil
+}

pkg/cwhub/itemremove.go

@@ -0,0 +1,139 @@
+package cwhub
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+	"slices"
+)
+
+// purge removes the actual config file that was downloaded.
+func (i *Item) purge() (bool, error) {
+	if !i.State.Downloaded {
+		log.Debugf("removing %s: not downloaded -- no need to remove", i.Name)
+		return false, nil
+	}
+
+	src, err := i.downloadPath()
+	if err != nil {
+		return false, err
+	}
+
+	if err := os.Remove(src); err != nil {
+		if os.IsNotExist(err) {
+			log.Debugf("%s doesn't exist, no need to remove", src)
+			return false, nil
+		}
+
+		return false, fmt.Errorf("while removing file: %w", err)
+	}
+
+	i.State.Downloaded = false
+	log.Infof("Removed source file [%s]: %s", i.Name, src)
+
+	return true, nil
+}
+
+// disable removes the install link, and optionally the downloaded content.
+func (i *Item) disable(purge bool, force bool) (bool, error) {
+	didRemove := true
+
+	err := i.removeInstallLink()
+	if os.IsNotExist(err) {
+		if !purge && !force {
+			link, _ := i.installPath()
+			return false, fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
+		}
+		didRemove = false
+	} else if err != nil {
+		return false, err
+	}
+
+	i.State.Installed = false
+
+	didPurge := false
+	if purge {
+		if didPurge, err = i.purge(); err != nil {
+			return didRemove, err
+		}
+	}
+
+	ret := didRemove || didPurge
+
+	return ret, nil
+}
+
+// Remove disables the item, optionally removing the downloaded content.
+func (i *Item) Remove(purge bool, force bool) (bool, error) {
+	if i.IsLocal() {
+		log.Warningf("%s is a local item, please delete manually", i.Name)
+		return false, nil
+	}
+
+	if i.State.Tainted && !force {
+		return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
+	}
+
+	if !i.State.Installed && !purge {
+		log.Infof("removing %s: not installed -- no need to remove", i.Name)
+		return false, nil
+	}
+
+	removed := false
+
+	descendants, err := i.descendants()
+	if err != nil {
+		return false, err
+	}
+
+	ancestors := i.Ancestors()
+
+	for _, sub := range i.SubItems() {
+		if !sub.State.Installed {
+			continue
+		}
+
+		// if the sub depends on a collection that is not a direct or indirect dependency
+		// of the current item, it is not removed
+		for _, subParent := range sub.Ancestors() {
+			if !purge && !subParent.State.Installed {
+				continue
+			}
+
+			// the ancestor that would block the removal of the sub item is also an ancestor
+			// of the item we are removing, so we don't want false warnings
+			// (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux,
+			// while we are removing crowdsecurity/sshd)
+			if slices.Contains(ancestors, subParent) {
+				continue
+			}
+
+			// the sub-item belongs to the item we are removing, but we already knew that
+			if subParent == i {
+				continue
+			}
+
+			if !slices.Contains(descendants, subParent) {
+				log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
+				continue
+			}
+		}
+
+		subRemoved, err := sub.Remove(purge, force)
+		if err != nil {
+			return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
+		}
+
+		removed = removed || subRemoved
+	}
+
+	didDisable, err := i.disable(purge, force)
+	if err != nil {
+		return false, fmt.Errorf("while removing %s: %w", i.Name, err)
+	}
+
+	removed = removed || didDisable
+
+	return removed, nil
+}

pkg/cwhub/helpers.go → pkg/cwhub/itemupgrade.go

@@ -14,156 +14,17 @@ import (
 
 	"github.com/enescakir/emoji"
 	log "github.com/sirupsen/logrus"
-	"slices"
 )
 
-// Install installs the item from the hub, downloading it if needed.
-func (i *Item) Install(force bool, downloadOnly bool) error {
-	if downloadOnly && i.State.Downloaded && i.State.UpToDate {
-		log.Infof("%s is already downloaded and up-to-date", i.Name)
-
-		if !force {
-			return nil
-		}
-	}
-
-	filePath, err := i.downloadLatest(force, true)
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", i.Name, err)
-	}
-
-	if downloadOnly {
-		log.Infof("Downloaded %s to %s", i.Name, filePath)
-		return nil
-	}
-
-	if err := i.enable(); err != nil {
-		return fmt.Errorf("while enabling %s: %w", i.Name, err)
-	}
-
-	log.Infof("Enabled %s", i.Name)
-
-	return nil
-}
-
-// descendants returns a list of all (direct or indirect) dependencies of the item.
-func (i *Item) descendants() ([]*Item, error) {
-	var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
-
-	collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
-		if item == nil {
-			return nil
-		}
-
-		if visited[item] {
-			return nil
-		}
-
-		visited[item] = true
-
-		for _, subItem := range item.SubItems() {
-			if subItem == i {
-				return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
-			}
-
-			*result = append(*result, subItem)
-
-			err := collectSubItems(subItem, visited, result)
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}
-
-	ret := []*Item{}
-	visited := map[*Item]bool{}
-
-	err := collectSubItems(i, visited, &ret)
-	if err != nil {
-		return nil, err
-	}
-
-	return ret, nil
-}
+// Upgrade downloads and applies the last version of the item from the hub.
+func (i *Item) Upgrade(force bool) (bool, error) {
+	updated := false
 
-// Remove disables the item, optionally removing the downloaded content.
-func (i *Item) Remove(purge bool, force bool) (bool, error) {
 	if i.IsLocal() {
-		return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
-	}
-
-	if i.State.Tainted && !force {
-		return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
-	}
-
-	if !i.State.Installed && !purge {
-		log.Infof("removing %s: not installed -- no need to remove", i.Name)
+		log.Infof("not upgrading %s: local item", i.Name)
 		return false, nil
 	}
 
-	removed := false
-
-	descendants, err := i.descendants()
-	if err != nil {
-		return false, err
-	}
-
-	ancestors := i.Ancestors()
-
-	for _, sub := range i.SubItems() {
-		if !sub.State.Installed {
-			continue
-		}
-
-		// if the sub depends on a collection that is not a direct or indirect dependency
-		// of the current item, it is not removed
-		for _, subParent := range sub.Ancestors() {
-			if !purge && !subParent.State.Installed {
-				continue
-			}
-
-			// the ancestor that would block the removal of the sub item is also an ancestor
-			// of the item we are removing, so we don't want false warnings
-			// (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux,
-			// while we are removing crowdsecurity/sshd)
-			if slices.Contains(ancestors, subParent) {
-				continue
-			}
-
-			// the sub-item belongs to the item we are removing, but we already knew that
-			if subParent == i {
-				continue
-			}
-
-			if !slices.Contains(descendants, subParent) {
-				log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
-				continue
-			}
-		}
-
-		subRemoved, err := sub.Remove(purge, force)
-		if err != nil {
-			return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
-		}
-
-		removed = removed || subRemoved
-	}
-
-	if err = i.disable(purge, force); err != nil {
-		return false, fmt.Errorf("while removing %s: %w", i.Name, err)
-	}
-
-	removed = true
-
-	return removed, nil
-}
-
-// Upgrade downloads and applies the last version of the item from the hub.
-func (i *Item) Upgrade(force bool) (bool, error) {
-	updated := false
-
 	if !i.State.Downloaded {
 		return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
 	}
@@ -192,8 +53,6 @@ func (i *Item) Upgrade(force bool) (bool, error) {
 	if !i.State.UpToDate {
 		if i.State.Tainted {
 			log.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
-		} else if i.IsLocal() {
-			log.Infof("%v %s is local", emoji.Prohibited, i.Name)
 		}
 	} else {
 		// a check on stdout is used while scripting to know if the hub has been upgraded
@@ -296,6 +155,9 @@ func (i *Item) fetch() ([]byte, error) {
 
 // download downloads the item from the hub and writes it to the hub directory.
 func (i *Item) download(overwrite bool) (string, error) {
+	if i.IsLocal() {
+		return "", fmt.Errorf("%s is local, can't download", i.Name)
+	}
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
 	if !overwrite {
 		if i.State.Tainted {

pkg/cwhub/helpers_test.go → pkg/cwhub/itemupgrade_test.go

@@ -13,20 +13,21 @@ import (
 func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 	hub := envSetup(t)
 
+	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
 	// fresh install of collection
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
+	require.False(t, item.State.Downloaded)
+	require.False(t, item.State.Installed)
 
-	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
 	require.NoError(t, item.Install(false, false))
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
+	require.True(t, item.State.Downloaded)
+	require.True(t, item.State.Installed)
+	require.True(t, item.State.UpToDate)
+	require.False(t, item.State.Tainted)
 
 	// This is the scenario that gets added in next version of collection
-	require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
+	require.Nil(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario"))
 
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
@@ -44,19 +45,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 
 	hub = getHubOrFail(t, hub.local, remote)
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
-
 	item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
+	require.True(t, item.State.Downloaded)
+	require.True(t, item.State.Installed)
+	require.False(t, item.State.UpToDate)
+	require.False(t, item.State.Tainted)
+
 	didUpdate, err := item.Upgrade(false)
 	require.NoError(t, err)
 	require.True(t, didUpdate)
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Downloaded)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed)
 }
 
 // Install a collection, disable a scenario.
@@ -64,19 +66,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
 func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	hub := envSetup(t)
 
+	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
 	// fresh install of collection
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, item.State.Downloaded)
+	require.False(t, item.State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 
-	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
 	require.NoError(t, item.Install(false, false))
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.True(t, item.State.Downloaded)
+	require.True(t, item.State.Installed)
+	require.True(t, item.State.UpToDate)
+	require.False(t, item.State.Tainted)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
 	item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
@@ -92,11 +95,12 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 
 	hub = getHubOrFail(t, hub.local, remote)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
+
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate)
 
 	hub, err = NewHub(hub.local, remote, true)
 	require.NoError(t, err, "failed to download index: %s", err)
@@ -107,7 +111,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
 	require.False(t, didUpdate)
 
 	hub = getHubOrFail(t, hub.local, remote)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 }
 
 // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
@@ -124,19 +128,20 @@ func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCf
 func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
 	hub := envSetup(t)
 
+	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
+
 	// fresh install of collection
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, item.State.Downloaded)
+	require.False(t, item.State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 
-	item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
 	require.NoError(t, item.Install(false, false))
 
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
-	require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.True(t, item.State.Downloaded)
+	require.True(t, item.State.Installed)
+	require.True(t, item.State.UpToDate)
+	require.False(t, item.State.Tainted)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 	assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
 
 	item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
@@ -152,12 +157,12 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
 
 	hub = getHubOrFail(t, hub.local, remote)
 	// scenario referenced by collection  was deleted hence, collection should be tainted
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
-	require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Downloaded) // this fails
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed)
+	require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate)
 
 	// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
 	// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
@@ -167,7 +172,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
 	hub, err = NewHub(hub.local, remote, true)
 	require.NoError(t, err, "failed to download index: %s", err)
 
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
 	hub = getHubOrFail(t, hub.local, remote)
 
 	item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
@@ -176,14 +181,14 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
 	require.True(t, didUpdate)
 
 	hub = getHubOrFail(t, hub.local, remote)
-	require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
-	require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
+	require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
+	require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed)
 }
 
 func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
 	t.Helper()
 
-	c := hub.Items[COLLECTIONS][collection]
+	c := hub.GetItem(COLLECTIONS, collection)
 	require.NoError(t, c.checkSubItemVersions())
 }
 

pkg/cwhub/sync.go

@@ -30,7 +30,7 @@ func linkTarget(path string) (string, error) {
 
 	_, err = os.Lstat(hubpath)
 	if os.IsNotExist(err) {
-		log.Infof("link target does not exist: %s -> %s", path, hubpath)
+		log.Warningf("link target does not exist: %s -> %s", path, hubpath)
 		return "", nil
 	}
 
@@ -226,7 +226,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 			if err != nil {
 				return err
 			}
-			h.Items[info.ftype][item.Name] = item
+			h.addItem(item)
 
 			return nil
 		}
@@ -246,7 +246,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 	// try to find which configuration item it is
 	log.Tracef("check [%s] of %s", info.fname, info.ftype)
 
-	for name, item := range h.Items[info.ftype] {
+	for _, item := range h.GetItemMap(info.ftype) {
 		if info.fname != item.FileName {
 			continue
 		}
@@ -287,8 +287,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
 			return err
 		}
 
-		h.Items[info.ftype][name] = item
-
 		return nil
 	}
 
@@ -395,7 +393,7 @@ func (h *Hub) localSync() error {
 
 	warnings := make([]string, 0)
 
-	for _, item := range h.Items[COLLECTIONS] {
+	for _, item := range h.GetItemMap(COLLECTIONS) {
 		// check for cyclic dependencies
 		subs, err := item.descendants()
 		if err != nil {

pkg/hubtest/coverage.go

@@ -19,12 +19,12 @@ type Coverage struct {
 }
 
 func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
-	if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
+	if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 {
 		return nil, fmt.Errorf("no parsers in hub index")
 	}
 
 	// populate from hub, iterate in alphabetical order
-	pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS])
+	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.PARSERS))
 	coverage := make([]Coverage, len(pkeys))
 
 	for i, name := range pkeys {
@@ -105,12 +105,12 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
 }
 
 func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
-	if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
+	if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0  {
 		return nil, fmt.Errorf("no scenarios in hub index")
 	}
 
 	// populate from hub, iterate in alphabetical order
-	pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS])
+	pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS))
 	coverage := make([]Coverage, len(pkeys))
 
 	for i, name := range pkeys {

pkg/hubtest/hubtest_item.go

@@ -146,7 +146,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 
-		if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
+		if hubParser := t.HubIndex.GetItem(cwhub.PARSERS, parser); hubParser != nil {
 			parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
@@ -232,7 +232,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 
-		if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
+		if hubScenario := t.HubIndex.GetItem(cwhub.SCENARIOS, scenario); hubScenario != nil {
 			scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
@@ -303,7 +303,7 @@ func (t *HubTestItem) InstallHub() error {
 			continue
 		}
 
-		if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
+		if hubPostOverflow := t.HubIndex.GetItem(cwhub.POSTOVERFLOWS, postoverflow); hubPostOverflow != nil {
 			postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
 			if err != nil {
 				return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)

pkg/metabase/container.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"context"
 	"fmt"
-	"runtime"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
@@ -93,14 +92,6 @@ func (c *Container) Create() error {
 		Tty:   true,
 		Env:   env,
 	}
-	os := runtime.GOOS
-	switch os {
-	case "linux":
-	case "windows", "darwin":
-		return fmt.Errorf("mac and windows are not supported yet")
-	default:
-		return fmt.Errorf("OS '%s' is not supported", os)
-	}
 
 	log.Infof("creating container '%s'", c.Name)
 	resp, err := c.CLI.ContainerCreate(ctx, dockerConfig, hostConfig, nil, nil, c.Name)

test/bats/20_hub.bats

@@ -119,3 +119,23 @@ teardown() {
     # this is used by the cron script to know if the hub was updated
     assert_output --partial "updated crowdsecurity/syslog-logs"
 }
+
+@test "cscli hub upgrade (with local items)" {
+    mkdir -p "$CONFIG_DIR/collections"
+    touch "$CONFIG_DIR/collections/foo.yaml"
+    rune -0 cscli hub upgrade
+    assert_stderr --partial "not upgrading foo.yaml: local item"
+}
+
+@test "cscli hub types" {
+    rune -0 cscli hub types -o raw
+    assert_line "parsers"
+    assert_line "postoverflows"
+    assert_line "scenarios"
+    assert_line "collections"
+    rune -0 cscli hub types -o human
+    rune -0 yq -o json <(output)
+    assert_json '["parsers","postoverflows","scenarios","collections"]'
+    rune -0 cscli hub types -o json
+    assert_json '["parsers","postoverflows","scenarios","collections"]'
+}

test/bats/20_hub_collections.bats

@@ -208,7 +208,7 @@ teardown() {
     assert_line 'type: collections'
     assert_line 'name: crowdsecurity/sshd'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: collections/crowdsecurity/sshd.yaml'
+    assert_line 'path: collections/crowdsecurity/sshd.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -226,7 +226,7 @@ teardown() {
     assert_line 'type: collections'
     assert_line 'name: crowdsecurity/sshd'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: collections/crowdsecurity/sshd.yaml'
+    assert_line 'path: collections/crowdsecurity/sshd.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -275,8 +275,9 @@ teardown() {
     rune -0 cscli collections remove crowdsecurity/sshd
     assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove'
 
-    rune -0 cscli collections remove crowdsecurity/sshd --purge
+    rune -0 cscli collections remove crowdsecurity/sshd --purge --debug
     assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no need to remove'
+    refute_stderr --partial 'Removed source file [crowdsecurity/sshd]'
 
     # install, then remove, check files
     rune -0 cscli collections install crowdsecurity/sshd

test/bats/20_hub_items.bats

@@ -99,7 +99,7 @@ teardown() {
     rune -0 jq -r '.path' <(output)
     rune -0 rm "$HUB_DIR/$(output)"
 
-    rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge
+    rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge --debug
     assert_stderr --partial "removing crowdsecurity/syslog-logs: not downloaded -- no need to remove"
 
     rune -0 cscli parsers remove crowdsecurity/linux --all --error --purge --force
@@ -147,3 +147,37 @@ teardown() {
     rune -0 cscli collections inspect hi-its-me -o json
     rune -0 jq -e '[.installed,.local]==[true,true]' <(output)
 }
+
+@test "a local item cannot be downloaded by cscli" {
+    rune -0 mkdir -p "$CONFIG_DIR/collections"
+    rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
+    rune -1 cscli collections install foobar.yaml
+    assert_stderr --partial "failed to download item: foobar.yaml is local, can't download"
+    rune -1 cscli collections install foobar.yaml --force
+    assert_stderr --partial "failed to download item: foobar.yaml is local, can't download"
+}
+
+@test "a local item cannot be removed by cscli" {
+    rune -0 mkdir -p "$CONFIG_DIR/collections"
+    rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
+    rune -0 cscli collections remove foobar.yaml
+    assert_stderr --partial "foobar.yaml is a local item, please delete manually"
+    rune -0 cscli collections remove foobar.yaml --purge
+    assert_stderr --partial "foobar.yaml is a local item, please delete manually"
+    rune -0 cscli collections remove foobar.yaml --force
+    assert_stderr --partial "foobar.yaml is a local item, please delete manually"
+    rune -0 cscli collections remove --all
+    assert_stderr --partial "foobar.yaml is a local item, please delete manually"
+    rune -0 cscli collections remove --all --purge
+    assert_stderr --partial "foobar.yaml is a local item, please delete manually"
+}
+
+@test "a dangling link is reported with a warning" {
+    rune -0 mkdir -p "$CONFIG_DIR/collections"
+    rune -0 ln -s /this/does/not/exist.yaml "$CONFIG_DIR/collections/foobar.yaml"
+    rune -0 cscli hub list
+    assert_stderr --partial "link target does not exist: $CONFIG_DIR/collections/foobar.yaml -> /this/does/not/exist.yaml"
+    rune -0 cscli hub list -o json
+    rune -0 jq '.collections' <(output)
+    assert_json '[]'
+}

test/bats/20_hub_parsers.bats

@@ -209,7 +209,7 @@ teardown() {
     assert_line 'stage: s01-parse'
     assert_line 'name: crowdsecurity/sshd-logs'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
+    assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -228,7 +228,7 @@ teardown() {
     assert_line 'name: crowdsecurity/sshd-logs'
     assert_line 'stage: s01-parse'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
+    assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -277,8 +277,9 @@ teardown() {
     rune -0 cscli parsers remove crowdsecurity/whitelists
     assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove"
 
-    rune -0 cscli parsers remove crowdsecurity/whitelists --purge
+    rune -0 cscli parsers remove crowdsecurity/whitelists --purge --debug
     assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no need to remove'
+    refute_stderr --partial 'Removed source file [crowdsecurity/whitelists]'
 
     # install, then remove, check files
     rune -0 cscli parsers install crowdsecurity/whitelists

test/bats/20_hub_postoverflows.bats

@@ -209,7 +209,7 @@ teardown() {
     assert_line 'stage: s00-enrich'
     assert_line 'name: crowdsecurity/rdns'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
+    assert_line 'path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -228,7 +228,7 @@ teardown() {
     assert_line 'name: crowdsecurity/rdns'
     assert_line 'stage: s00-enrich'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
+    assert_line 'path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -277,8 +277,9 @@ teardown() {
     rune -0 cscli postoverflows remove crowdsecurity/rdns
     assert_stderr --partial 'removing crowdsecurity/rdns: not installed -- no need to remove'
 
-    rune -0 cscli postoverflows remove crowdsecurity/rdns --purge
+    rune -0 cscli postoverflows remove crowdsecurity/rdns --purge --debug
     assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no need to remove'
+    refute_stderr --partial 'Removed source file [crowdsecurity/rdns]'
 
     # install, then remove, check files
     rune -0 cscli postoverflows install crowdsecurity/rdns

test/bats/20_hub_scenarios.bats

@@ -209,7 +209,7 @@ teardown() {
     assert_line 'type: scenarios'
     assert_line 'name: crowdsecurity/ssh-bf'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml'
+    assert_line 'path: scenarios/crowdsecurity/ssh-bf.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -227,7 +227,7 @@ teardown() {
     assert_line 'type: scenarios'
     assert_line 'name: crowdsecurity/ssh-bf'
     assert_line 'author: crowdsecurity'
-    assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml'
+    assert_line 'path: scenarios/crowdsecurity/ssh-bf.yaml'
     assert_line 'installed: false'
     refute_line --partial 'Current metrics:'
 
@@ -276,8 +276,9 @@ teardown() {
     rune -0 cscli scenarios remove crowdsecurity/ssh-bf
     assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove"
 
-    rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge --debug
     assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no need to remove'
+    refute_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]'
 
     # install, then remove, check files
     rune -0 cscli scenarios install crowdsecurity/ssh-bf

test/bin/preload-hub-items

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck disable=SC1091
+. "${THIS_DIR}/../.environment.sh"
+
+# pre-download everything but don't install anything
+
+echo -n "Purging existing hub..."
+
+types=$("$CSCLI" hub types -o raw)
+
+for itemtype in $types; do
+    "$CSCLI" "${itemtype}" delete --all --error --purge --force
+done
+
+echo " done."
+
+echo -n "Pre-downloading Hub content..."
+
+for itemtype in $types; do
+    ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name')
+    if [[ -n "${ALL_ITEMS}" ]]; then
+        #shellcheck disable=SC2086
+        "$CSCLI" "$itemtype" install \
+            $ALL_ITEMS \
+            --download-only \
+            --error
+    fi
+done
+
+# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
+# so we have to delete the links manually, and leave the downloaded files in place
+
+for itemtype in $types; do
+    "$CSCLI" "$itemtype" delete --all --error
+done
+
+echo " done."

test/lib/config/config-global

@@ -54,50 +54,6 @@ remove_init_data() {
 
 # we need a separate function for initializing config when testing package
 # because we want to test the configuration as well
-preload_hub_items() {
-    # pre-download everything but don't install anything
-    # each test can install what it needs
-
-    echo "Purging existing hub..."
-
-    "$CSCLI" parsers delete --all --error --purge --force
-    "$CSCLI" scenarios delete --all --error --purge --force
-    "$CSCLI" postoverflows delete --all --error --purge --force
-    "$CSCLI" collections delete --all --error --purge --force
-
-    echo "Pre-downloading hub content..."
-
-    #shellcheck disable=SC2046
-    "$CSCLI" collections install \
-        $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" parsers install \
-        $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" scenarios install \
-        $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" postoverflows install \
-        $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
-        --download-only \
-        --error
-
-    # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
-    # so we have to delete the links manually, and leave the downloaded files in place
-
-    "$CSCLI" parsers delete --all --error
-    "$CSCLI" scenarios delete --all --error
-    "$CSCLI" postoverflows delete --all --error
-}
 
 make_init_data() {
     ./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
@@ -108,7 +64,7 @@ make_init_data() {
     # when installed packages are always using sqlite, so no need to regenerate
     # local credz for sqlite
 
-    preload_hub_items
+    ./bin/preload-hub-items
 
     [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto
 
@@ -145,7 +101,6 @@ load_init_data() {
     ./instance-db restore "${LOCAL_INIT_DIR}/database"
 }
 
-
 # ---------------------------
 
 [[ $# -lt 1 ]] && about

test/lib/config/config-local

@@ -101,51 +101,6 @@ config_generate() {
     ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml"
 }
 
-preload_hub_items() {
-    # pre-download everything but don't install anything
-    # each test can install what it needs
-
-    echo "Purging existing hub..."
-
-    "$CSCLI" parsers delete --all --error --purge --force
-    "$CSCLI" scenarios delete --all --error --purge --force
-    "$CSCLI" postoverflows delete --all --error --purge --force
-    "$CSCLI" collections delete --all --error --purge --force
-
-    echo "Pre-downloading hub content..."
-
-    #shellcheck disable=SC2046
-    "$CSCLI" collections install \
-        $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" parsers install \
-        $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" scenarios install \
-        $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
-        --download-only \
-        --error
-
-    #shellcheck disable=SC2046
-    "$CSCLI" postoverflows install \
-        $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
-        --download-only \
-        --error
-
-    # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
-    # so we have to delete the links manually, and leave the downloaded files in place
-
-    "$CSCLI" parsers delete --all --error
-    "$CSCLI" scenarios delete --all --error
-    "$CSCLI" postoverflows delete --all --error
-}
-
 make_init_data() {
     ./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
 
@@ -163,7 +118,7 @@ make_init_data() {
     "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
     "$CSCLI" --warning hub update
 
-    preload_hub_items
+    ./bin/preload-hub-items
 
     mkdir -p "$LOCAL_INIT_DIR"
 
@@ -197,7 +152,6 @@ load_init_data() {
     ./instance-db restore "${LOCAL_INIT_DIR}/database"
 }
 
-
 # ---------------------------
 
 [[ $# -lt 1 ]] && about

test/lib/setup_file.sh

@@ -243,8 +243,16 @@ export -f assert_stderr_line
 hub_purge_all() {
     local CONFIG_DIR
     CONFIG_DIR=$(dirname "$CONFIG_YAML")
-    rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/*
-    rm -rf "$CONFIG_DIR"/hub/collections/* "$CONFIG_DIR"/hub/parsers/*/* "$CONFIG_DIR"/hub/scenarios/* "$CONFIG_DIR"/hub/postoverflows/*
+    rm -rf \
+        "$CONFIG_DIR"/collections/* \
+        "$CONFIG_DIR"/parsers/*/* \
+        "$CONFIG_DIR"/scenarios/* \
+        "$CONFIG_DIR"/postoverflows/*
+    rm -rf \
+        "$CONFIG_DIR"/hub/collections/* \
+        "$CONFIG_DIR"/hub/parsers/*/* \
+        "$CONFIG_DIR"/hub/scenarios/* \
+        "$CONFIG_DIR"/hub/postoverflows/*
     local DATA_DIR
     DATA_DIR=$(config_get .config_paths.data_dir)
     # should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),