Merge branch 'hub-1.5.6' into coraza_poc_acquis

.github/workflows/bats.yml

@@ -15,12 +15,14 @@ on:
   push:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - "README.md"
   pull_request:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - "README.md"
@@ -31,7 +33,7 @@ jobs:
 
   # Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
   # CI, but they pass when run on devs' machines or in the release checks. We
-  # disable them here by default. Remove the if..false to enable them.
+  # disable them here by default. Remove if...false to enable them.
 
   mariadb:
     uses: ./.github/workflows/bats-mysql.yml

.github/workflows/codeql-analysis.yml

@@ -16,11 +16,13 @@ on:
   push:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
   pull_request:
     # The branches below must be a subset of the branches above
     branches:
       - master
+      - hub-1.5.6
       - releases/**
   schedule:
     - cron: '15 16 * * 2'

.github/workflows/docker-tests.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - 'README.md'
   pull_request:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - 'README.md'

.github/workflows/go-tests-windows.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - 'README.md'
   pull_request:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - 'README.md'

.github/workflows/go-tests.yml

@@ -9,12 +9,14 @@ on:
   push:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - 'README.md'
   pull_request:
     branches:
       - master
+      - hub-1.5.6
       - releases/**
     paths-ignore:
       - 'README.md'

cmd/crowdsec-cli/config_backup.go

@@ -45,7 +45,7 @@ func backupHub(dirPath string) error {
 				continue
 			}
 
-			//for the local/tainted ones, we backup the full file
+			//for the local/tainted ones, we back up the full file
 			if v.Tainted || v.IsLocal() || !v.UpToDate {
 				//we need to backup stages for parsers
 				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {

cmd/crowdsec-cli/dashboard.go

@@ -37,7 +37,7 @@ var (
 
 	forceYes bool
 
-	/*informations needed to setup a random password on user's behalf*/
+	// information needed to set up a random password on user's behalf
 )
 
 func NewDashboardCmd() *cobra.Command {

cmd/crowdsec-cli/hub.go

@@ -12,7 +12,7 @@ import (
 )
 
 func NewHubCmd() *cobra.Command {
-	var cmdHub = &cobra.Command{
+	cmdHub := &cobra.Command{
 		Use:   "hub [action]",
 		Short: "Manage hub index",
 		Long: `Hub management
@@ -54,7 +54,16 @@ func runHubList(cmd *cobra.Command, args []string) error {
 		log.Info(line)
 	}
 
-	err = ListItems(hub, color.Output, cwhub.ItemTypes, nil, true, false, all)
+	items := make(map[string][]*cwhub.Item)
+
+	for _, itemType := range cwhub.ItemTypes {
+		items[itemType], err = selectItems(hub, itemType, nil, !all)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = listItems(color.Output, cwhub.ItemTypes, items)
 	if err != nil {
 		return err
 	}
@@ -63,7 +72,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
 }
 
 func NewHubListCmd() *cobra.Command {
-	var cmdHubList = &cobra.Command{
+	cmdHubList := &cobra.Command{
 		Use:               "list [-a]",
 		Short:             "List all installed configurations",
 		Args:              cobra.ExactArgs(0),
@@ -95,7 +104,7 @@ func runHubUpdate(cmd *cobra.Command, args []string) error {
 }
 
 func NewHubUpdateCmd() *cobra.Command {
-	var cmdHubUpdate = &cobra.Command{
+	cmdHubUpdate := &cobra.Command{
 		Use:   "update",
 		Short: "Download the latest index (catalog of available configurations)",
 		Long: `
@@ -129,6 +138,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
 		}
 
 		updated := 0
+
 		log.Infof("Upgrading %s", itemType)
 		for _, item := range items {
 			didUpdate, err := item.Upgrade(force)
@@ -146,7 +156,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
 }
 
 func NewHubUpgradeCmd() *cobra.Command {
-	var cmdHubUpgrade = &cobra.Command{
+	cmdHubUpgrade := &cobra.Command{
 		Use:   "upgrade",
 		Short: "Upgrade all configurations to their latest version",
 		Long: `

cmd/crowdsec-cli/itemcommands.go

@@ -10,6 +10,7 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/coalesce"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
 type cmdHelp struct {
@@ -61,7 +62,9 @@ cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
 		listHelp: cmdHelp{
 			example: `cscli parsers list
 cscli parsers list -a
-cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+
+List only enabled parsers unless "-a" or names are specified.`,
 		},
 	},
 	"postoverflows": {
@@ -91,7 +94,9 @@ cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
 		listHelp: cmdHelp{
 			example: `cscli postoverflows list
 cscli postoverflows list -a
-cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
+
+List only enabled postoverflows unless "-a" or names are specified.`,
 		},
 	},
 	"scenarios": {
@@ -121,7 +126,9 @@ cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
 		listHelp: cmdHelp{
 			example: `cscli scenarios list
 cscli scenarios list -a
-cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
+
+List only enabled scenarios unless "-a" or names are specified.`,
 		},
 	},
 	"waap-rules": {
@@ -211,7 +218,9 @@ cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
 		listHelp: cmdHelp{
 			example: `cscli collections list
 cscli collections list -a
-cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`,
+cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
+
+List only enabled collections unless "-a" or names are specified.`,
 		},
 	},
 }
@@ -340,7 +349,12 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
 		}
 
 		if all {
-			items, err := hub.GetInstalledItems(it.name)
+			getter := hub.GetInstalledItems
+			if purge {
+				getter = hub.GetAllItems
+			}
+
+			items, err := getter(it.name)
 			if err != nil {
 				return err
 			}
@@ -555,7 +569,7 @@ func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string)
 			if item == nil {
 				return fmt.Errorf("can't find '%s' in %s", name, it.name)
 			}
-			if err = InspectItem(hub, item, !noMetrics); err != nil {
+			if err = InspectItem(item, !noMetrics); err != nil {
 				return err
 			}
 		}
@@ -603,7 +617,14 @@ func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) err
 			return err
 		}
 
-		if err = ListItems(hub, color.Output, []string{it.name}, args, false, true, all); err != nil {
+		items := make(map[string][]*cwhub.Item)
+
+		items[it.name], err = selectItems(hub, it.name, args, !all)
+		if err != nil {
+			return err
+		}
+
+		if err = listItems(color.Output, []string{it.name}, items); err != nil {
 			return err
 		}
 

cmd/crowdsec-cli/items.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"sort"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -15,7 +14,8 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) {
+// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
+func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
 	itemNames := hub.GetItemNames(itemType)
 
 	notExist := []string{}
@@ -37,33 +37,27 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b
 		installedOnly = false
 	}
 
-	if installedOnly {
+	items := make([]*cwhub.Item, 0, len(itemNames))
-		installed := []string{}
+
-		for _, item := range itemNames {
+	for _, itemName := range itemNames {
-			if hub.GetItem(itemType, item).Installed {
+		item := hub.GetItem(itemType, itemName)
-				installed = append(installed, item)
+		if installedOnly && !item.Installed {
-			}
+			continue
 		}
-		return installed, nil
+
+		items = append(items, item)
 	}
-	return itemNames, nil
+
+	cwhub.SortItemSlice(items)
+
+	return items, nil
 }
 
-func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
+func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
-	items := make(map[string][]string)
-	for _, itemType := range itemTypes {
-		selected, err := selectItems(hub, itemType, args, !all)
-		if err != nil {
-			return err
-		}
-		sort.Strings(selected)
-		items[itemType] = selected
-	}
-
 	switch csConfig.Cscli.Output {
 	case "human":
 		for _, itemType := range itemTypes {
-			listHubItemTable(hub, out, "\n"+strings.ToUpper(itemType), itemType, items[itemType])
+			listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
 		}
 	case "json":
 		type itemHubStatus struct {
@@ -79,8 +73,8 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string,
 		for _, itemType := range itemTypes {
 			// empty slice in case there are no items of this type
 			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
-			for i, itemName := range items[itemType] {
+
-				item := hub.GetItem(itemType, itemName)
+			for i, item := range items[itemType] {
 				status, emo := item.Status()
 				hubStatus[itemType][i] = itemHubStatus{
 					Name:         item.Name,
@@ -92,38 +86,35 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string,
 				}
 			}
 		}
+
 		x, err := json.MarshalIndent(hubStatus, "", " ")
 		if err != nil {
 			return fmt.Errorf("failed to unmarshal: %w", err)
 		}
+
 		out.Write(x)
 	case "raw":
 		csvwriter := csv.NewWriter(out)
 
-		if showHeader {
+		header := []string{"name", "status", "version", "description"}
-			header := []string{"name", "status", "version", "description"}
+		if len(itemTypes) > 1 {
-			if showType {
+			header = append(header, "type")
-				header = append(header, "type")
-			}
-			err := csvwriter.Write(header)
-			if err != nil {
-				return fmt.Errorf("failed to write header: %s", err)
-			}
 		}
+
+		if err := csvwriter.Write(header); err != nil {
+			return fmt.Errorf("failed to write header: %s", err)
+		}
+
 		for _, itemType := range itemTypes {
-			for _, itemName := range items[itemType] {
+			for _, item := range items[itemType] {
-				item := hub.GetItem(itemType, itemName)
 				status, _ := item.Status()
-				if item.LocalVersion == "" {
-					item.LocalVersion = "n/a"
-				}
 				row := []string{
 					item.Name,
 					status,
 					item.LocalVersion,
 					item.Description,
 				}
-				if showType {
+				if len(itemTypes) > 1 {
 					row = append(row, itemType)
 				}
 				if err := csvwriter.Write(row); err != nil {
@@ -139,7 +130,7 @@ func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string,
 	return nil
 }
 
-func InspectItem(hub *cwhub.Hub, item *cwhub.Item, showMetrics bool) error {
+func InspectItem(item *cwhub.Item, showMetrics bool) error {
 	switch csConfig.Cscli.Output {
 	case "human", "raw":
 		enc := yaml.NewEncoder(os.Stdout)

cmd/crowdsec-cli/require/require.go

@@ -74,6 +74,7 @@ func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
 	remote := &cwhub.RemoteHubCfg {
 		Branch: branch,
 		URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
+		// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
 		IndexPath: ".index.json",
 	}
 

cmd/crowdsec-cli/support.go

@@ -129,9 +129,18 @@ func collectOSInfo() ([]byte, error) {
 }
 
 func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
+	var err error
+
 	out := bytes.NewBuffer(nil)
 	log.Infof("Collecting %s list", itemType)
-	if err := ListItems(hub, out, []string{itemType}, []string{}, false, true, false); err != nil {
+
+	items := make(map[string][]*cwhub.Item)
+
+	if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
+
+	if err := listItems(out, []string{itemType}, items); err != nil {
 		log.Warnf("could not collect %s list: %s", itemType, err)
 	}
 	return out.Bytes()

cmd/crowdsec-cli/utils_table.go

@@ -11,14 +11,13 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func listHubItemTable(hub *cwhub.Hub, out io.Writer, title string, itemType string, itemNames []string) {
+func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
 	t := newLightTable(out)
 	t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	for itemName := range itemNames {
+	for _, item := range items {
-		item := hub.GetItem(itemType, itemNames[itemName])
 		status, emo := item.Status()
 		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.LocalVersion, item.LocalPath)
 	}

docker/docker_start.sh

@@ -101,19 +101,23 @@ register_bouncer() {
 # $2 can be install, remove, upgrade
 # $3 is a list of object names separated by space
 cscli_if_clean() {
+    local itemtype="$1"
+    local action="$2"
+    local objs=$3
+    shift 3
     # loop over all objects
-    for obj in $3; do
+    for obj in $objs; do
-        if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
+        if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
-            echo "Object $1/$obj is tainted, skipping"
+            echo "Object $itemtype/$obj is tainted, skipping"
         else
 #            # Too verbose? Only show errors if not in debug mode
 #            if [ "$DEBUG" != "true" ]; then
 #                error_only=--error
 #            fi
             error_only=""
-            echo "Running: cscli $error_only $1 $2 \"$obj\""
+            echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
             # shellcheck disable=SC2086
-            cscli $error_only "$1" "$2" "$obj"
+            cscli $error_only "$itemtype" "$action" "$obj" "$@"
         fi
     done
 }

docker/test/tests/test_hub_collections.py

@@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor):
         cs.wait_for_log([
             # f'*collections install "{it1}"*'
             # f'*collections install "{it2}"*'
-            f'*Enabled collections : {it1}*',
+            f'*Enabled collections: {it1}*',
-            f'*Enabled collections : {it2}*',
+            f'*Enabled collections: {it2}*',
         ])
 
 
@@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor):
         assert it not in items
         logs = cs.log_lines()
         # check that there was no attempt to install
-        assert not any(f'Enabled collections : {it}' in line for line in logs)
+        assert not any(f'Enabled collections: {it}' in line for line in logs)
 
 
 # already done in bats, prividing here as example of a somewhat complex test
@@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
         # implicit check for tainted=False
         assert items[coll]['status'] == 'enabled'
         cs.wait_for_log([
-            f'*Enabled collections : {coll}*',
+            f'*Enabled collections: {coll}*',
         ])
 
         scenario = 'crowdsecurity/http-crawl-non_statics'

docker/test/tests/test_hub_scenarios.py

@@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor):
     }
     with crowdsec(flavor=flavor, environment=env) as cs:
         cs.wait_for_log([
-            f'*scenarios install "{it1}*"',
+            f'*scenarios install "{it1}"*',
-            f'*scenarios install "{it2}*"',
+            f'*scenarios install "{it2}"*',
             "*Starting processing data*"
         ])
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)

pkg/acquisition/modules/cloudwatch/cloudwatch_test.go

@@ -199,7 +199,7 @@ stream_regexp: test_bad[0-9]+`),
 			},
 			expectedResLen: 0,
 		},
-		// require a group name that does exist and contains a stream in which we gonna put events
+		// require a group name that does exist and contains a stream in which we are going to put events
 		{
 			name: "group_exists_stream_exists_has_events",
 			config: []byte(`

pkg/apiserver/apic.go

@@ -213,7 +213,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con
 	}
 
 	// The watcher will be authenticated by the RoundTripper the first time it will call CAPI
-	// Explicit authentication will provoke an useless supplementary call to CAPI
+	// Explicit authentication will provoke a useless supplementary call to CAPI
 	scenarios, err := ret.FetchScenariosListFromDB()
 	if err != nil {
 		return ret, fmt.Errorf("get scenario in db: %w", err)

pkg/apiserver/apic_test.go

@@ -572,7 +572,7 @@ func TestAPICWhitelists(t *testing.T) {
 					&modelscapi.GetDecisionsStreamResponseDeletedItem{
 						Decisions: []string{
 							"9.9.9.9", // This is already present in DB
-							"9.1.9.9", // This not present in DB
+							"9.1.9.9", // This is not present in DB
 						},
 						Scope: ptr.Of("Ip"),
 					}, // This is already present in DB
@@ -734,7 +734,7 @@ func TestAPICPullTop(t *testing.T) {
 					&modelscapi.GetDecisionsStreamResponseDeletedItem{
 						Decisions: []string{
 							"9.9.9.9", // This is already present in DB
-							"9.1.9.9", // This not present in DB
+							"9.1.9.9", // This is not present in DB
 						},
 						Scope: ptr.Of("Ip"),
 					}, // This is already present in DB

pkg/csconfig/config.go

@@ -23,7 +23,7 @@ var defaultDataDir = "/var/lib/crowdsec/data/"
 
 // Config contains top-level defaults -> overridden by configuration file -> overridden by CLI flags
 type Config struct {
-	//just a path to ourself :p
+	//just a path to ourselves :p
 	FilePath     *string             `yaml:"-"`
 	Self         []byte              `yaml:"-"`
 	Common       *CommonCfg          `yaml:"common,omitempty"`

pkg/csplugin/broker.go

@@ -110,7 +110,7 @@ loop:
 			pb.addProfileAlert(profileAlert)
 
 		case pluginName := <-pb.watcher.PluginEvents:
-			// this can be ran in goroutine, but then locks will be needed
+			// this can be run in goroutine, but then locks will be needed
 			pluginMutex.Lock()
 			log.Tracef("going to deliver %d alerts to plugin %s", len(pb.alertsByPluginName[pluginName]), pluginName)
 			tmpAlerts := pb.alertsByPluginName[pluginName]
@@ -139,7 +139,7 @@ loop:
 					pb.Kill()
 					break loop
 				case pluginName := <-pb.watcher.PluginEvents:
-					// this can be ran in goroutine, but then locks will be needed
+					// this can be run in goroutine, but then locks will be needed
 					pluginMutex.Lock()
 					log.Tracef("going to deliver %d alerts to plugin %s", len(pb.alertsByPluginName[pluginName]), pluginName)
 					tmpAlerts := pb.alertsByPluginName[pluginName]
@@ -206,7 +206,7 @@ func (pb *PluginBroker) loadConfig(path string) error {
 	return err
 }
 
-// checks whether every notification in profile has it's own config file
+// checks whether every notification in profile has its own config file
 func (pb *PluginBroker) verifyPluginConfigsWithProfile() error {
 	for _, profileCfg := range pb.profileConfigs {
 		for _, pluginName := range profileCfg.Notifications {
@@ -219,7 +219,7 @@ func (pb *PluginBroker) verifyPluginConfigsWithProfile() error {
 	return nil
 }
 
-// check whether each plugin in profile has it's own binary
+// check whether each plugin in profile has its own binary
 func (pb *PluginBroker) verifyPluginBinaryWithProfile() error {
 	for _, profileCfg := range pb.profileConfigs {
 		for _, pluginName := range profileCfg.Notifications {

pkg/cwhub/cwhub.go

@@ -10,5 +10,5 @@ import (
 )
 
 var hubClient = &http.Client{
-	Timeout: 10 * time.Second,
+	Timeout: 120 * time.Second,
 }

pkg/cwhub/cwhub_test.go

@@ -48,27 +48,20 @@ func testHub(t *testing.T, update bool) *Hub {
 	err = os.MkdirAll(local.InstallDataDir, 0o700)
 	require.NoError(t, err)
 
-	index, err := os.Create(local.HubIndexFile)
+	err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644)
 	require.NoError(t, err)
 
-	_, err = index.WriteString(`{}`)
-	require.NoError(t, err)
-
-	index.Close()
-
 	t.Cleanup(func() {
 		os.RemoveAll(tmpDir)
 	})
 
-	var hub *Hub
-
 	remote := &RemoteHubCfg{
 		Branch:      "master",
 		URLTemplate: mockURLTemplate,
 		IndexPath:   ".index.json",
 	}
 
-	hub, err = NewHub(local, remote, update)
+	hub, err := NewHub(local, remote, update)
 	require.NoError(t, err)
 
 	return hub

pkg/cwhub/enable.go

@@ -10,12 +10,42 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-// enable creates a symlink between actual config file at hub.HubDir and hub.ConfigDir
+// installLink returns the location of the symlink to the actual config file (eg. /etc/crowdsec/collections/xyz.yaml)
-// Handles collections recursively
+func (i *Item) installLink() string {
-func (i *Item) enable() error {
+	return filepath.Join(i.hub.local.InstallDir, i.Type, i.Stage, i.FileName)
-	parentDir := filepath.Clean(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/")
+}
 
-	// create directories if needed
+// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir
+func (i *Item) createInstallLink() error {
+	dest, err := filepath.Abs(i.installLink())
+	if err != nil {
+		return err
+	}
+
+	destDir := filepath.Dir(dest)
+	if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
+		return fmt.Errorf("while creating %s: %w", destDir, err)
+	}
+
+	if _, err = os.Lstat(dest); !os.IsNotExist(err) {
+		log.Infof("%s already exists.", dest)
+		return nil
+	}
+
+	src, err := filepath.Abs(filepath.Join(i.hub.local.HubDir, i.RemotePath))
+	if err != nil {
+		return err
+	}
+
+	if err = os.Symlink(src, dest); err != nil {
+		return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
+	}
+
+	return nil
+}
+
+// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items
+func (i *Item) enable() error {
 	if i.Installed {
 		if i.Tainted {
 			return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
@@ -32,40 +62,14 @@ func (i *Item) enable() error {
 		}
 	}
 
-	if _, err := os.Stat(parentDir); os.IsNotExist(err) {
-		log.Infof("%s doesn't exist, create", parentDir)
-
-		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
-			return fmt.Errorf("while creating directory: %w", err)
-		}
-	}
-
-	// install sub-items if any
 	for _, sub := range i.SubItems() {
 		if err := sub.enable(); err != nil {
 			return fmt.Errorf("while installing %s: %w", sub.Name, err)
 		}
 	}
 
-	// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
+	if err := i.createInstallLink(); err != nil {
-	if _, err := os.Lstat(parentDir + "/" + i.FileName); !os.IsNotExist(err) {
+		return err
-		log.Infof("%s already exists.", parentDir+"/"+i.FileName)
-		return nil
-	}
-
-	// hub.ConfigDir + target.RemotePath
-	srcPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
-	if err != nil {
-		return fmt.Errorf("while getting source path: %w", err)
-	}
-
-	dstPath, err := filepath.Abs(parentDir + "/" + i.FileName)
-	if err != nil {
-		return fmt.Errorf("while getting destination path: %w", err)
-	}
-
-	if err = os.Symlink(srcPath, dstPath); err != nil {
-		return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
 	}
 
 	log.Infof("Enabled %s: %s", i.Type, i.Name)
@@ -76,15 +80,65 @@ func (i *Item) enable() error {
 
 // purge removes the actual config file that was downloaded
 func (i *Item) purge() error {
-	itempath := i.hub.local.HubDir + "/" + i.RemotePath
+	if !i.Downloaded {
+		log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
+		return nil
+	}
+
+	src := filepath.Join(i.hub.local.HubDir, i.RemotePath)
+
+	if err := os.Remove(src); err != nil {
+		if os.IsNotExist(err) {
+			log.Debugf("%s doesn't exist, no need to remove", src)
+			return nil
+		}
 
-	// disable hub file
-	if err := os.Remove(itempath); err != nil {
 		return fmt.Errorf("while removing file: %w", err)
 	}
 
 	i.Downloaded = false
-	log.Infof("Removed source file [%s]: %s", i.Name, itempath)
+	log.Infof("Removed source file [%s]: %s", i.Name, src)
+
+	return nil
+}
+
+func (i *Item) removeInstallLink() error {
+	syml, err := filepath.Abs(i.installLink())
+	if err != nil {
+		return err
+	}
+
+	stat, err := os.Lstat(syml)
+	if err != nil {
+		return err
+	}
+
+	// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
+	if stat.Mode()&os.ModeSymlink == 0 {
+		log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
+		return fmt.Errorf("%s isn't managed by hub", i.Name)
+	}
+
+	hubpath, err := os.Readlink(syml)
+	if err != nil {
+		return fmt.Errorf("while reading symlink: %w", err)
+	}
+
+	src, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
+	if err != nil {
+		return err
+	}
+
+	if hubpath != src {
+		log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
+		return fmt.Errorf("%s isn't managed by hub", i.Name)
+	}
+
+	if err := os.Remove(syml); err != nil {
+		return fmt.Errorf("while removing symlink: %w", err)
+	}
+
+	log.Infof("Removed symlink [%s]: %s", i.Name, syml)
 
 	return nil
 }
@@ -92,92 +146,19 @@ func (i *Item) purge() error {
 // disable removes the symlink to the downloaded content, also removes the content if purge is true
 func (i *Item) disable(purge bool, force bool) error {
 	// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
-	var err error
+	err := i.removeInstallLink()
-
-	// already disabled, noop unless purge
-	if !i.Installed {
-		if purge {
-			if err = i.purge(); err != nil {
-				return err
-			}
-		}
-
-		return nil
-	}
-
-	if i.IsLocal() {
-		return fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
-	}
-
-	if i.Tainted && !force {
-		return fmt.Errorf("%s is tainted, use '--force' to overwrite", i.Name)
-	}
-
-	// disable sub-items if any - it's a collection
-	for _, sub := range i.SubItems() {
-		// check if the item doesn't belong to another collection before removing it
-		removeSub := true
-
-		for _, collection := range sub.BelongsToCollections {
-			if collection != i.Name {
-				removeSub = false
-				break
-			}
-		}
-
-		if removeSub {
-			if err = sub.disable(purge, force); err != nil {
-				return fmt.Errorf("while disabling %s: %w", sub.Name, err)
-			}
-		} else {
-			log.Infof("%s was not removed because it belongs to another collection", sub.Name)
-		}
-	}
-
-	syml, err := filepath.Abs(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/" + i.FileName)
-	if err != nil {
-		return err
-	}
-
-	stat, err := os.Lstat(syml)
 	if os.IsNotExist(err) {
-		// we only accept to "delete" non existing items if it's a forced purge
 		if !purge && !force {
-			return fmt.Errorf("can't delete %s: %s doesn't exist", i.Name, syml)
+			return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLink())
 		}
-	} else {
+	} else if err != nil {
-		// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
+		return err
-		if stat.Mode()&os.ModeSymlink == 0 {
-			log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
-			return fmt.Errorf("%s isn't managed by hub", i.Name)
-		}
-
-		hubpath, err := os.Readlink(syml)
-		if err != nil {
-			return fmt.Errorf("while reading symlink: %w", err)
-		}
-
-		absPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
-		if err != nil {
-			return fmt.Errorf("while abs path: %w", err)
-		}
-
-		if hubpath != absPath {
-			log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, absPath)
-			return fmt.Errorf("%s isn't managed by hub", i.Name)
-		}
-
-		if err = os.Remove(syml); err != nil {
-			return fmt.Errorf("while removing symlink: %w", err)
-		}
-
-		log.Infof("Removed symlink [%s]: %s", i.Name, syml)
 	}
 
 	i.Installed = false
 
 	if purge {
-		if err = i.purge(); err != nil {
+		if err := i.purge(); err != nil {
 			return err
 		}
 	}

pkg/cwhub/helpers.go

@@ -17,12 +17,13 @@ import (
 
 	"github.com/enescakir/emoji"
 	log "github.com/sirupsen/logrus"
+	"slices"
 )
 
 // Install installs the item from the hub, downloading it if needed
 func (i *Item) Install(force bool, downloadOnly bool) error {
 	if downloadOnly && i.Downloaded && i.UpToDate {
-		log.Warningf("%s is already downloaded and up-to-date", i.Name)
+		log.Infof("%s is already downloaded and up-to-date", i.Name)
 
 		if !force {
 			return nil
@@ -51,22 +52,70 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
 	return nil
 }
 
-// Remove disables the item, optionally removing the downloaded content
+// allDependencies return a list of all dependencies and sub-dependencies of the item
-func (i *Item) Remove(purge bool, forceAction bool) (bool, error) {
+func (i *Item) allDependencies() []*Item {
-	removed := false
+	var deps []*Item
 
-	if !i.Downloaded {
+	for _, dep := range i.SubItems() {
-		log.Infof("removing %s: not downloaded -- no removal required", i.Name)
+		if dep == i {
-		return false, nil
+			log.Errorf("circular dependency detected: %s depends on %s", dep.Name, i.Name)
+			continue
+		}
+
+		deps = append(deps, dep.allDependencies()...)
+	}
+
+	return append(deps, i)
+}
+
+// Remove disables the item, optionally removing the downloaded content
+func (i *Item) Remove(purge bool, force bool) (bool, error) {
+	if i.IsLocal() {
+		return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
+	}
+
+	if i.Tainted && !force {
+		return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
 	}
 
 	if !i.Installed && !purge {
-		log.Infof("removing %s: already uninstalled", i.Name)
+		log.Infof("removing %s: not installed -- no need to remove", i.Name)
 		return false, nil
 	}
 
-	if err := i.disable(purge, forceAction); err != nil {
+	removed := false
-		return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
+
+	allDeps := i.allDependencies()
+
+	for _, sub := range i.SubItems() {
+		if !sub.Installed {
+			continue
+		}
+
+		// if the other collection(s) are direct or indirect dependencies of the current one, it's good to go
+		// log parent collections
+		for _, subParent := range sub.parentCollections() {
+			if subParent == i {
+				continue
+			}
+
+			if !slices.Contains(allDeps, subParent) {
+				log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
+				continue
+			}
+		}
+
+		subRemoved, err := sub.Remove(purge, force)
+		if err != nil {
+			return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
+		}
+
+		removed = removed || subRemoved
+	}
+
+	err := i.disable(purge, force)
+	if err != nil {
+		return false, fmt.Errorf("while removing %s: %w", i.Name, err)
 	}
 
 	// XXX: should take the value from disable()
@@ -171,14 +220,46 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error {
 	return nil
 }
 
-func (i *Item) download(overwrite bool) error {
+// fetch downloads the item from the hub, verifies the hash and returns the body
+func (i *Item) fetch() ([]byte, error) {
 	url, err := i.hub.remote.urlTo(i.RemotePath)
 	if err != nil {
-		return fmt.Errorf("failed to build hub item request: %w", err)
+		return nil, fmt.Errorf("failed to build hub item request: %w", err)
 	}
 
-	tdir := i.hub.local.HubDir
+	resp, err := hubClient.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("while downloading %s: %w", url, err)
+	}
+	defer resp.Body.Close()
 
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("while downloading %s: %w", url, err)
+	}
+
+	hash := sha256.New()
+	if _, err = hash.Write(body); err != nil {
+		return nil, fmt.Errorf("while hashing %s: %w", i.Name, err)
+	}
+
+	meow := hex.EncodeToString(hash.Sum(nil))
+	if meow != i.Versions[i.Version].Digest {
+		log.Errorf("Downloaded version doesn't match index, please 'hub update'")
+		log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
+
+		return nil, fmt.Errorf("invalid download hash for %s", i.Name)
+	}
+
+	return body, nil
+}
+
+// download downloads the item from the hub and writes it to the hub directory
+func (i *Item) download(overwrite bool) error {
 	// if user didn't --force, don't overwrite local, tainted, up-to-date files
 	if !overwrite {
 		if i.Tainted {
@@ -192,56 +273,29 @@ func (i *Item) download(overwrite bool) error {
 		}
 	}
 
-	resp, err := hubClient.Get(url)
+	body, err := i.fetch()
 	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", url, err)
+		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
 	}
 
-	body, err := io.ReadAll(resp.Body)
+	tdir := i.hub.local.HubDir
-	if err != nil {
-		return fmt.Errorf("while downloading %s: %w", url, err)
-	}
-
-	hash := sha256.New()
-	if _, err = hash.Write(body); err != nil {
-		return fmt.Errorf("while hashing %s: %w", i.Name, err)
-	}
-
-	meow := hex.EncodeToString(hash.Sum(nil))
-	if meow != i.Versions[i.Version].Digest {
-		log.Errorf("Downloaded version doesn't match index, please 'hub update'")
-		log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
-
-		return fmt.Errorf("invalid download hash for %s", i.Name)
-	}
 
 	//all good, install
-	//check if parent dir exists
-	tmpdirs := strings.Split(tdir+"/"+i.RemotePath, "/")
-	parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
 
-	// ensure that target file is within target dir
+	finalPath, err := filepath.Abs(filepath.Join(tdir, i.RemotePath))
-	finalPath, err := filepath.Abs(tdir + "/" + i.RemotePath)
 	if err != nil {
-		return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+i.RemotePath, err)
+		return err
 	}
 
+	// ensure that target file is within target dir
 	if !strings.HasPrefix(finalPath, tdir) {
 		return fmt.Errorf("path %s escapes %s, abort", i.RemotePath, tdir)
 	}
 
-	// check dir
+	parentDir := filepath.Dir(finalPath)
-	if _, err = os.Stat(parentDir); os.IsNotExist(err) {
-		log.Debugf("%s doesn't exist, create", parentDir)
 
-		if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
+	if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
-			return fmt.Errorf("while creating parent directories: %w", err)
+		return fmt.Errorf("while creating %s: %w", parentDir, err)
-		}
 	}
 
 	// check actual file
@@ -252,15 +306,8 @@ func (i *Item) download(overwrite bool) error {
 		log.Infof("%s: OK", i.Name)
 	}
 
-	f, err := os.Create(tdir + "/" + i.RemotePath)
+	if err = os.WriteFile(finalPath, body, 0o644); err != nil {
-	if err != nil {
+		return fmt.Errorf("while writing %s: %w", finalPath, err)
-		return fmt.Errorf("while opening file: %w", err)
-	}
-	defer f.Close()
-
-	_, err = f.Write(body)
-	if err != nil {
-		return fmt.Errorf("while writing file: %w", err)
 	}
 
 	i.Downloaded = true

pkg/cwhub/hub_test.go

@@ -69,5 +69,5 @@ func TestDownloadIndex(t *testing.T) {
 	}
 
 	err = hub.remote.downloadIndex("/does/not/exist/index.json")
-	cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:")
+	cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:")
 }

pkg/cwhub/items.go

@@ -3,6 +3,8 @@ package cwhub
 import (
 	"encoding/json"
 	"fmt"
+	"sort"
+	"strings"
 
 	"github.com/Masterminds/semver/v3"
 	"github.com/enescakir/emoji"
@@ -206,6 +208,21 @@ func (i *Item) logMissingSubItems() {
 	}
 }
 
+func (i *Item) parentCollections() []*Item {
+	ret := make([]*Item, 0)
+
+	for _, parentName := range i.BelongsToCollections {
+		parent := i.hub.GetItem(COLLECTIONS, parentName)
+		if parent == nil {
+			continue
+		}
+
+		ret = append(ret, parent)
+	}
+
+	return ret
+}
+
 // Status returns the status of the item as a string and an emoji
 // ie. "enabled,update-available" and emoji.Warning
 func (i *Item) Status() (string, emoji.Emoji) {
@@ -270,7 +287,7 @@ func (i *Item) versionStatus() int {
 }
 
 // validPath returns true if the (relative) path is allowed for the item
-// dirNmae: the directory name (ie. crowdsecurity)
+// dirNname: the directory name (ie. crowdsecurity)
 // fileName: the filename (ie. apache2-logs.yaml)
 func (i *Item) validPath(dirName, fileName string) bool {
 	return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
@@ -303,6 +320,25 @@ func (h *Hub) GetItemNames(itemType string) []string {
 	return names
 }
 
+// GetAllItems returns a slice of all the items, installed or not
+func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
+	items, ok := h.Items[itemType]
+	if !ok {
+		return nil, fmt.Errorf("no %s in the hub index", itemType)
+	}
+
+	ret := make([]*Item, len(items))
+
+	idx := 0
+
+	for _, item := range items {
+		ret[idx] = item
+		idx++
+	}
+
+	return ret, nil
+}
+
 // GetInstalledItems returns the list of installed items
 func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
 	items, ok := h.Items[itemType]
@@ -336,3 +372,10 @@ func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
 
 	return retStr, nil
 }
+
+// SortItemSlice sorts a slice of items by name, case insensitive
+func SortItemSlice(items []*Item) {
+	sort.Slice(items, func(i, j int) bool {
+		return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
+	})
+}

pkg/cwhub/remote.go

@@ -70,18 +70,11 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error {
 		return nil
 	}
 
-	file, err := os.Create(localPath)
+	if err = os.WriteFile(localPath, body, 0o644); err != nil {
-	if err != nil {
+		return fmt.Errorf("failed to write hub index: %w", err)
-		return fmt.Errorf("while opening hub index file: %w", err)
-	}
-	defer file.Close()
-
-	wsize, err := file.Write(body)
-	if err != nil {
-		return fmt.Errorf("while writing hub index file: %w", err)
 	}
 
-	log.Infof("Wrote index to %s, %d bytes", localPath, wsize)
+	log.Infof("Wrote index to %s, %d bytes", localPath, len(body))
 
 	return nil
 }

pkg/cwhub/sync.go

@@ -156,10 +156,8 @@ func sortedVersions(raw []string) ([]string, error) {
 }
 
 func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
-	var (
+	local := false
-		local   bool
+	hubpath := ""
-		hubpath string
-	)
 
 	if err != nil {
 		log.Debugf("while syncing hub dir: %s", err)
@@ -401,9 +399,12 @@ func (h *Hub) syncDir(dir string) ([]string, error) {
 
 	// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
 	for _, scan := range ItemTypes {
+		// cpath: top-level item directory, either downloaded or installed items.
+		// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...
 		cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
 		if err != nil {
 			log.Errorf("failed %s: %s", cpath, err)
+			continue
 		}
 
 		// explicit check for non existing directory, avoid spamming log.Debug

pkg/hubtest/parser_assert.go

@@ -369,6 +369,11 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
 	}
 
 	sort.Strings(parsers)
+
+	if len(parsers) == 0 {
+		return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
+	}
+
 	lastParser := parsers[len(parsers)-1]
 
 	for idx, result := range pdump[lastStage][lastParser] {

pkg/leakybucket/bucket.go

@@ -30,13 +30,13 @@ type Leaky struct {
 	//the limiter is what holds the proper "leaky aspect", it determines when/if we can pour objects
 	Limiter         rate.RateLimiter `json:"-"`
 	SerializedState rate.Lstate
-	//Queue is used to held the cache of objects in the bucket, it is used to know 'how many' objects we have in buffer.
+	//Queue is used to hold the cache of objects in the bucket, it is used to know 'how many' objects we have in buffer.
 	Queue *Queue
 	//Leaky buckets are receiving message through a chan
 	In chan *types.Event `json:"-"`
 	//Leaky buckets are pushing their overflows through a chan
 	Out chan *Queue `json:"-"`
-	// shared for all buckets (the idea is to kill this afterwards)
+	// shared for all buckets (the idea is to kill this afterward)
 	AllOut chan types.Event `json:"-"`
 	//max capacity (for burst)
 	Capacity int
@@ -332,7 +332,7 @@ func LeakRoutine(leaky *Leaky) error {
 
 			}
 			if leaky.logger.Level >= log.TraceLevel {
-				/*don't sdump if it's not going to printed, it's expensive*/
+				/*don't sdump if it's not going to be printed, it's expensive*/
 				leaky.logger.Tracef("Overflow event: %s", spew.Sdump(types.Event{Overflow: alert}))
 			}
 

pkg/metabase/api.go

@@ -79,7 +79,7 @@ func (h *MBClient) Do(method string, route string, body interface{}) (interface{
 	return Success, Error, err
 }
 
-// Set set headers as key:value
+// Set headers as key:value
 func (h *MBClient) Set(key string, value string) {
 	h.CTX = h.CTX.Set(key, value)
 }

test/ansible/README.md

@@ -107,7 +107,7 @@ To test with Vagrant, you need to:
   `/var/lib/libvirt/images/*VAGRANT*`
 
 The above steps are automated in the script `./prepare-run` (requires bash
->=4.4). It takes an enviroment file, and optionally a list of directories with
+>=4.4). It takes an environment file, and optionally a list of directories with
 vagrant configurations. With a single parameter, it loops over all the
 directories in alphabetical order, excluding those in the `experimental`
 directory. Watch out for running VMs if you break the loop by hand.

test/ansible/requirements.yml

@@ -8,7 +8,8 @@ roles:
     name: geerlingguy.postgresql
   # these should be included as dependencies of crowdsecurity.testing, but sometime are not
   - src: geerlingguy.repo-epel
-  - src: gantsign.golang
+  - src: gantsign.ansible-role-golang
+    name: gantsign.golang
 
 collections:
   - name: ansible.posix

test/bats/00_wait_for.bats

@@ -0,0 +1,71 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+@test "run a command and capture its stdout" {
+    run -0 wait-for seq 1 3
+    assert_output - <<-EOT
+	1
+	2
+	3
+	EOT
+}
+
+@test "run a command and capture its stderr" {
+    rune -0 wait-for sh -c 'seq 1 3 >&2'
+    assert_stderr - <<-EOT
+	1
+	2
+	3
+	EOT
+}
+
+@test "run a command until a pattern is found in stdout" {
+    run -0 wait-for --out "1[12]0" seq 1 200
+    assert_line --index 0 "1"
+    assert_line --index -1 "110"
+    refute_line "111"
+}
+
+@test "run a command until a pattern is found in stderr" {
+    rune -0 wait-for --err "10" sh -c 'seq 1 20 >&2'
+    assert_stderr - <<-EOT
+	1
+	2
+	3
+	4
+	5
+	6
+	7
+	8
+	9
+	10
+	EOT
+}
+
+@test "run a command with timeout (no match)" {
+    # when the process is terminated without a match, it returns
+    # 256 - 15 (SIGTERM) = 241
+    rune -241 wait-for --timeout 0.1 --out "10" sh -c 'echo 1; sleep 3; echo 2'
+    assert_line 1
+    # there may be more, but we don't care
+}
+
+@test "run a command with timeout (match)" {
+    # when the process is terminated with a match, return code is 128
+    rune -128 wait-for --timeout .4 --out "2" sh -c 'echo 1; sleep .1; echo 2; echo 3; echo 4; sleep 10'
+    assert_output - <<-EOT
+	1
+	2
+	EOT
+}
+

test/bats/01_crowdsec.bats

@@ -24,28 +24,22 @@ teardown() {
 #----------
 
 @test "crowdsec (usage)" {
-    rune -0 timeout 2s "${CROWDSEC}" -h
+    rune -0 wait-for --out "Usage of " "${CROWDSEC}" -h
-    assert_stderr_line --regexp "Usage of .*:"
+    rune -0 wait-for --out "Usage of " "${CROWDSEC}" --help
-
-    rune -0 timeout 2s "${CROWDSEC}" --help
-    assert_stderr_line --regexp "Usage of .*:"
 }
 
 @test "crowdsec (unknown flag)" {
-    rune -2 timeout 2s "${CROWDSEC}" --foobar
+    rune -0 wait-for --err "flag provided but not defined: -foobar" "$CROWDSEC" --foobar
-    assert_stderr_line "flag provided but not defined: -foobar"
-    assert_stderr_line --regexp "Usage of .*"
 }
 
 @test "crowdsec (unknown argument)" {
-    rune -2 timeout 2s "${CROWDSEC}" trololo
+    rune -0 wait-for --err "argument provided but not defined: trololo" "${CROWDSEC}" trololo
-    assert_stderr_line "argument provided but not defined: trololo"
-    assert_stderr_line --regexp "Usage of .*"
 }
 
 @test "crowdsec (no api and no agent)" {
-    rune -1 timeout 2s "${CROWDSEC}" -no-api -no-cs
+    rune -0 wait-for \
-    assert_stderr_line --partial "You must run at least the API Server or crowdsec"
+        --err "You must run at least the API Server or crowdsec" \
+        "${CROWDSEC}" -no-api -no-cs
 }
 
 @test "crowdsec - print error on exit" {
@@ -57,18 +51,20 @@ teardown() {
 
 @test "crowdsec - default logging configuration (empty/missing common section)" {
     config_set '.common={}'
-    rune -124 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "Starting processing data"
 
     config_set 'del(.common)'
-    rune -124 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
     refute_output
-    assert_stderr --partial "Starting processing data"
 }
 
 @test "CS_LAPI_SECRET not strong enough" {
-    CS_LAPI_SECRET=foo rune -1 timeout 2s "${CROWDSEC}"
+    CS_LAPI_SECRET=foo rune -1 wait-for "${CROWDSEC}"
     assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
 }
 
@@ -138,8 +134,8 @@ teardown() {
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
     rm -f "$ACQUIS_YAML"
 
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -1 wait-for "${CROWDSEC}"
-    assert_stderr_line --partial "acquis.yaml: no such file or directory"
+    assert_stderr --partial "acquis.yaml: no such file or directory"
 }
 
 @test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" {
@@ -151,7 +147,7 @@ teardown() {
     rm -f "$ACQUIS_DIR"
 
     config_set '.common.log_media="stdout"'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -1 wait-for "${CROWDSEC}"
     # check warning
     assert_stderr --partial "no acquisition file found"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
@@ -167,13 +163,15 @@ teardown() {
     config_set '.crowdsec_service.acquisition_dir=""'
 
     config_set '.common.log_media="stdout"'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -1 wait-for "${CROWDSEC}"
     # check warning
     assert_stderr --partial "no acquisition_path or acquisition_dir specified"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
+    config_set '.common.log_media="stdout"'
+
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
     config_set '.crowdsec_service.acquisition_path=""'
 
@@ -181,13 +179,15 @@ teardown() {
     mkdir -p "$ACQUIS_DIR"
     mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
 
-    rune -124 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
 
     # now, if foo.yaml is empty instead, there won't be valid datasources.
 
     cat /dev/null >"$ACQUIS_DIR"/foo.yaml
 
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -1 wait-for "${CROWDSEC}"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
@@ -212,9 +212,10 @@ teardown() {
 	  type: syslog
 	EOT
 
-    rune -124 timeout 2s env PATH='' "${CROWDSEC}"
     #shellcheck disable=SC2016
-    assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
+    rune -0 wait-for \
+        --err 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in ' \
+        env PATH='' "${CROWDSEC}"
 
     # if all datasources are disabled, crowdsec should exit
 
@@ -222,7 +223,7 @@ teardown() {
     rm -f "$ACQUIS_YAML"
     config_set '.crowdsec_service.acquisition_path=""'
 
-    rune -1 timeout 2s env PATH='' "${CROWDSEC}"
+    rune -1 wait-for env PATH='' "${CROWDSEC}"
     assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 

test/bats/01_cscli.bats

@@ -249,36 +249,29 @@ teardown() {
     # we check for the presence of some objects. There may be others when we
     # use $PACKAGE_TESTING, so the order is not important.
 
+    rune -0 cscli parsers install crowdsecurity/whitelists
+    rune -0 cscli scenarios install crowdsecurity/asterisk_user_enum
+    rune -0 cscli collections install crowdsecurity/sshd
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+
     rune -0 cscli hub list -o human
-    assert_line --regexp '^ crowdsecurity/linux'
+    assert_line --regexp '^ crowdsecurity/whitelists'
+    assert_line --regexp '^ crowdsecurity/asterisk_user_enum'
     assert_line --regexp '^ crowdsecurity/sshd'
-    assert_line --regexp '^ crowdsecurity/dateparse-enrich'
+    assert_line --regexp '^ crowdsecurity/rdns'
-    assert_line --regexp '^ crowdsecurity/geoip-enrich'
-    assert_line --regexp '^ crowdsecurity/sshd-logs'
-    assert_line --regexp '^ crowdsecurity/syslog-logs'
-    assert_line --regexp '^ crowdsecurity/ssh-bf'
-    assert_line --regexp '^ crowdsecurity/ssh-slow-bf'
 
     rune -0 cscli hub list -o raw
-    assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$'
+    assert_line --regexp '^crowdsecurity/whitelists,enabled,.*'
-    assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$'
+    assert_line --regexp '^crowdsecurity/asterisk_user_enum,enabled,.*'
-    assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$'
+    assert_line --regexp '^crowdsecurity/sshd,enabled,.*'
-    assert_line --regexp '^crowdsecurity/geoip-enrich,enabled,[0-9]+\.[0-9]+,"Populate event with geoloc info : as, country, coords, source range.",parsers$'
+    assert_line --regexp '^crowdsecurity/rdns,enabled,.*'
-    assert_line --regexp '^crowdsecurity/sshd-logs,enabled,[0-9]+\.[0-9]+,Parse openSSH logs,parsers$'
-    assert_line --regexp '^crowdsecurity/syslog-logs,enabled,[0-9]+\.[0-9]+,,parsers$'
-    assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$'
-    assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$'
 
     rune -0 cscli hub list -o json
-    rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name' <(output)
+    rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name, .postoverflows[].name' <(output)
-    assert_line 'crowdsecurity/linux'
+    assert_line 'crowdsecurity/whitelists'
+    assert_line 'crowdsecurity/asterisk_user_enum'
     assert_line 'crowdsecurity/sshd'
-    assert_line 'crowdsecurity/dateparse-enrich'
+    assert_line 'crowdsecurity/rdns'
-    assert_line 'crowdsecurity/geoip-enrich'
-    assert_line 'crowdsecurity/sshd-logs'
-    assert_line 'crowdsecurity/syslog-logs'
-    assert_line 'crowdsecurity/ssh-bf'
-    assert_line 'crowdsecurity/ssh-slow-bf'
 }
 
 @test "cscli support dump (smoke test)" {
@@ -287,8 +280,17 @@ teardown() {
 }
 
 @test "cscli explain" {
-    rune -0 cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog --crowdsec "$CROWDSEC"
+    line="Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4"
+
+    rune -0 cscli parsers install crowdsecurity/syslog-logs
+    rune -0 cscli collections install crowdsecurity/sshd
+
+    rune -0 cscli explain --log "$line" --type syslog --only-successful-parsers --crowdsec "$CROWDSEC"
     assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt
+
+    rune -0 cscli parsers remove --all --purge
+    rune -1 cscli explain --log "$line" --type syslog --crowdsec "$CROWDSEC"
+    assert_stderr --partial "unable to load parser dump result: no parser found. Please install the appropriate parser and retry"
 }
 
 @test 'Allow variable expansion and literal $ characters in passwords' {

test/bats/02_nolapi.bats

@@ -24,21 +24,23 @@ teardown() {
 #----------
 
 @test "test without -no-api flag" {
-    rune -124 timeout 2s "${CROWDSEC}"
+    config_set '.common.log_media="stdout"'
-    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+    rune -0 wait-for \
+        --err "CrowdSec Local API listening" \
+        "${CROWDSEC}"
 }
 
 @test "crowdsec should not run without LAPI (-no-api flag)" {
-    # really needs 4 secs on slow boxes
+    config_set '.common.log_media="stdout"'
-    rune -1 timeout 4s "${CROWDSEC}" -no-api
+    rune -1 wait-for "${CROWDSEC}" -no-api
 }
 
 @test "crowdsec should not run without LAPI (no api.server in configuration file)" {
     config_disable_lapi
     config_log_stderr
-    # really needs 4 secs on slow boxes
+    rune -0 wait-for \
-    rune -1 timeout 4s "${CROWDSEC}"
+        --err "crowdsec local API is disabled" \
-    assert_stderr --partial "crowdsec local API is disabled"
+        "${CROWDSEC}"
 }
 
 @test "capi status shouldn't be ok without api.server" {

test/bats/03_noagent.bats

@@ -23,20 +23,25 @@ teardown() {
 #----------
 
 @test "with agent: test without -no-cs flag" {
-    rune -124 timeout 2s "${CROWDSEC}"
+    config_set '.common.log_media="stdout"'
-    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
 }
 
 @test "no agent: crowdsec LAPI should run (-no-cs flag)" {
-    rune -124 timeout 2s "${CROWDSEC}" -no-cs
+    config_set '.common.log_media="stdout"'
+    rune -0 wait-for \
+        --err "CrowdSec Local API listening" \
+        "${CROWDSEC}" -no-cs
 }
 
 @test "no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" {
     config_disable_agent
     config_log_stderr
-    rune -124 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-
+        --err "crowdsec agent is disabled" \
-    assert_stderr --partial "crowdsec agent is disabled"
+        "${CROWDSEC}"
 }
 
 @test "no agent: cscli config show" {

test/bats/04_capi.bats

@@ -22,6 +22,10 @@ setup() {
 @test "cscli capi status" {
     config_enable_capi
     rune -0 cscli capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX
+    rune -1 cscli capi status
+    assert_stderr --partial "no scenarios installed, abort"
+
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
     rune -0 cscli capi status
     assert_stderr --partial "Loaded credentials from"
     assert_stderr --partial "Trying to authenticate with username"

test/bats/04_nocapi.bats

@@ -25,16 +25,17 @@ teardown() {
 @test "without capi: crowdsec LAPI should run without capi (-no-capi flag)" {
     config_set '.common.log_media="stdout"'
 
-    rune -124 timeout 1s "${CROWDSEC}" -no-capi
+    rune -0 wait-for \
-    assert_stderr --partial "Communication with CrowdSec Central API disabled from args"
+        --err "Communication with CrowdSec Central API disabled from args" \
+        "${CROWDSEC}" -no-capi
 }
 
 @test "without capi: crowdsec LAPI should still work" {
     config_disable_capi
     config_set '.common.log_media="stdout"'
-    rune -124 timeout 1s "${CROWDSEC}"
+    rune -0 wait-for \
-    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+        --err "push and pull to Central API disabled" \
-    assert_stderr --partial "push and pull to Central API disabled"
+        "${CROWDSEC}"
 }
 
 @test "without capi: cscli capi status -> fail" {
@@ -47,10 +48,7 @@ teardown() {
 @test "no capi: cscli config show" {
     config_disable_capi
     rune -0 cscli config show -o human
-    assert_output --partial "Global:"
+    assert_output --regexp "Global:.*Crowdsec.*cscli:.*Local API Server:"
-    assert_output --partial "cscli:"
-    assert_output --partial "Crowdsec:"
-    assert_output --partial "Local API Server:"
 }
 
 @test "no agent: cscli config backup" {

test/bats/05_config_yaml_local.bats

@@ -56,28 +56,28 @@ teardown() {
     # disable the agent or we'll need to patch api client credentials too
     rune -0 config_disable_agent
     ./instance-crowdsec start
-    rune -0 ./bin/wait-for-port -q 8080
+    rune -0 wait-for-port -q 8080
     ./instance-crowdsec stop
-    rune -1 ./bin/wait-for-port -q 8080
+    rune -1 wait-for-port -q 8080
 
     echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local"
 
     ./instance-crowdsec start
-    rune -0 ./bin/wait-for-port -q 8083
+    rune -0 wait-for-port -q 8083
-    rune -1 ./bin/wait-for-port -q 8080
+    rune -1 wait-for-port -q 8080
     ./instance-crowdsec stop
 
     rm -f "${CONFIG_YAML}.local"
     ./instance-crowdsec start
-    rune -1 ./bin/wait-for-port -q 8083
+    rune -1 wait-for-port -q 8083
-    rune -0 ./bin/wait-for-port -q 8080
+    rune -0 wait-for-port -q 8080
 }
 
 @test "local_api_credentials.yaml.local" {
     rune -0 config_disable_agent
     echo "{'api':{'server':{'listen_uri':127.0.0.1:8083}}}" >"${CONFIG_YAML}.local"
     ./instance-crowdsec start
-    rune -0 ./bin/wait-for-port -q 8083
+    rune -0 wait-for-port -q 8083
 
     rune -1 cscli decisions list
     echo "{'url':'http://127.0.0.1:8083'}" >"${LOCAL_API_CREDENTIALS}.local"
@@ -127,6 +127,9 @@ teardown() {
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
     echo -e "---\nfilename: ${tmpfile}\nlabels:\n  type: syslog\n" >>"${ACQUIS_YAML}"
 
+    rune -0 cscli collections install crowdsecurity/sshd
+    rune -0 cscli parsers install crowdsecurity/syslog-logs
+
     ./instance-crowdsec start
     sleep .5
     fake_log >>"${tmpfile}"

test/bats/07_setup.bats

@@ -507,18 +507,16 @@ update-notifier-motd.timer              enabled enabled
 
 @test "cscli setup install-hub (dry run)" {
     # it's not installed
-    rune -0 cscli collections list -o json
+    rune -0 cscli collections inspect crowdsecurity/apache2 -o json
-    rune -0 jq -r '.collections[].name' <(output)
+    rune -0 jq -e '.installed == false' <(output)
-    refute_line "crowdsecurity/apache2"
 
     # we install it
     rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}'
     assert_output 'dry-run: would install collection crowdsecurity/apache2'
 
     # still not installed
-    rune -0 cscli collections list -o json
+    rune -0 cscli collections inspect crowdsecurity/apache2 -o json
-    rune -0 jq -r '.collections[].name' <(output)
+    rune -0 jq -e '.installed == false' <(output)
-    refute_line "crowdsecurity/apache2"
 
     # same with dependencies
     rune -0 cscli collections remove --all
@@ -528,18 +526,16 @@ update-notifier-motd.timer              enabled enabled
 
 @test "cscli setup install-hub (dry run: install multiple collections)" {
     # it's not installed
-    rune -0 cscli collections list -o json
+    rune -0 cscli collections inspect crowdsecurity/apache2 -o json
-    rune -0 jq -r '.collections[].name' <(output)
+    rune -0 jq -e '.installed == false' <(output)
-    refute_line "crowdsecurity/apache2"
 
     # we install it
     rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}'
     assert_output 'dry-run: would install collection crowdsecurity/apache2'
 
     # still not installed
-    rune -0 cscli collections list -o json
+    rune -0 cscli collections inspect crowdsecurity/apache2 -o json
-    rune -0 jq -r '.collections[].name' <(output)
+    rune -0 jq -e '.installed == false' <(output)
-    refute_line "crowdsecurity/apache2"
 }
 
 @test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" {

test/bats/13_capi_whitelists.bats

@@ -18,6 +18,7 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
+    config_set '.common.log_media="stdout"'
     config_set '.api.server.capi_whitelists_path=strenv(CAPI_WHITELISTS_YAML)'
 }
 
@@ -28,38 +29,51 @@ teardown() {
 #----------
 
 @test "capi_whitelists: file missing" {
-    rune -1 timeout 1s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "capi whitelist file '$CAPI_WHITELISTS_YAML' does not exist"
+        --err "capi whitelist file '$CAPI_WHITELISTS_YAML' does not exist" \
+        "${CROWDSEC}"
 }
 
 @test "capi_whitelists: error on open" {
     echo > "$CAPI_WHITELISTS_YAML"
     chmod 000 "$CAPI_WHITELISTS_YAML"
-    rune -1 timeout 1s "${CROWDSEC}"
+    if is_package_testing; then
-    assert_stderr --partial "while opening capi whitelist file: open $CAPI_WHITELISTS_YAML: permission denied"
+        rune -0 wait-for \
+            --err "while parsing capi whitelist file .*: empty file" \
+            "${CROWDSEC}"
+    else
+        rune -0 wait-for \
+            --err "while opening capi whitelist file: open $CAPI_WHITELISTS_YAML: permission denied" \
+            "${CROWDSEC}"
+    fi
 }
 
 @test "capi_whitelists: empty file" {
     echo > "$CAPI_WHITELISTS_YAML"
-    rune -1 timeout 1s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': empty file"
+        --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': empty file" \
+        "${CROWDSEC}"
 }
 
 @test "capi_whitelists: empty lists" {
     echo '{"ips": [], "cidrs": []}' > "$CAPI_WHITELISTS_YAML"
-    rune -124 timeout 1s "${CROWDSEC}"
+    rune -0 wait-for \
+        --err "Starting processing data" \
+        "${CROWDSEC}"
 }
 
 @test "capi_whitelists: bad ip" {
     echo '{"ips": ["blahblah"], "cidrs": []}' > "$CAPI_WHITELISTS_YAML"
-    rune -1 timeout 1s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid IP address: blahblah"
+        --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid IP address: blahblah" \
+        "${CROWDSEC}"
 }
 
 @test "capi_whitelists: bad cidr" {
     echo '{"ips": [], "cidrs": ["blahblah"]}' > "$CAPI_WHITELISTS_YAML"
-    rune -1 timeout 1s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid CIDR address: blahblah"
+        --err "while parsing capi whitelist file '$CAPI_WHITELISTS_YAML': invalid CIDR address: blahblah" \
+        "${CROWDSEC}"
 }
 
 @test "capi_whitelists: file with ip and cidr values" {

test/bats/20_hub.bats

@@ -6,8 +6,8 @@ set -u
 setup_file() {
     load "../lib/setup_file.sh"
     ./instance-data load
-    HUB_DIR=$(config_get '.config_paths.hub_dir')
+    INDEX_PATH=$(config_get '.config_paths.index_path')
-    export HUB_DIR
+    export INDEX_PATH
     CONFIG_DIR=$(config_get '.config_paths.config_dir')
     export CONFIG_DIR
 }
@@ -20,7 +20,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_purge_all
     hub_strip_index
 }
 
@@ -31,13 +30,15 @@ teardown() {
 #----------
 
 @test "cscli hub list" {
+    hub_purge_all
+
     # no items
     rune -0 cscli hub list
     assert_output --regexp ".*PARSERS.*POSTOVERFLOWS.*SCENARIOS.*COLLECTIONS.*"
     rune -0 cscli hub list -o json
     assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}'
     rune -0 cscli hub list -o raw
-    refute_output
+    assert_output 'name,status,version,description,type'
 
     # some items
     rune -0 cscli parsers install crowdsecurity/whitelists
@@ -49,38 +50,54 @@ teardown() {
     rune -0 cscli hub list -o raw
     assert_output --partial 'crowdsecurity/whitelists'
     assert_output --partial 'crowdsecurity/telnet-bf'
-    refute_output --partial 'crowdsecurity/linux'
+    refute_output --partial 'crowdsecurity/iptables'
 
     # all items
     rune -0 cscli hub list -a
-    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/linux.*"
+    assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*"
     rune -0 cscli hub list -a -o json
     rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output)
     rune -0 cscli hub list -a -o raw
     assert_output --partial 'crowdsecurity/whitelists'
     assert_output --partial 'crowdsecurity/telnet-bf'
-    assert_output --partial 'crowdsecurity/linux'
+    assert_output --partial 'crowdsecurity/iptables'
 }
 
 @test "missing reference in hub index" {
-    new_hub=$(jq <"$HUB_DIR/.index.json" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")')
+    new_hub=$(jq <"$INDEX_PATH" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")')
-    echo "$new_hub" >"$HUB_DIR/.index.json"
+    echo "$new_hub" >"$INDEX_PATH"
     rune -0 cscli hub list --error
     assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb"
     assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql"
 }
 
 @test "cscli hub update" {
-    #XXX: todo
+    rm -f "$INDEX_PATH"
-    :
+    rune -0 cscli hub update
+    assert_stderr --partial "Wrote index to $INDEX_PATH"
+    rune -0 cscli hub update
+    assert_stderr --partial "hub index is up to date"
 }
 
 @test "cscli hub upgrade" {
-    #XXX: todo
+    rune -0 cscli hub upgrade
-    :
+    assert_stderr --partial "Upgrading parsers"
-}
+    assert_stderr --partial "Upgraded 0 parsers"
+    assert_stderr --partial "Upgrading postoverflows"
+    assert_stderr --partial "Upgraded 0 postoverflows"
+    assert_stderr --partial "Upgrading scenarios"
+    assert_stderr --partial "Upgraded 0 scenarios"
+    assert_stderr --partial "Upgrading collections"
+    assert_stderr --partial "Upgraded 0 collections"
 
-@test "cscli hub upgrade --force" {
+    rune -0 cscli parsers install crowdsecurity/syslog-logs
-    #XXX: todo
+    rune -0 cscli hub upgrade
-    :
+    assert_stderr --partial "crowdsecurity/syslog-logs: up-to-date"
+
+    rune -0 cscli hub upgrade --force
+    assert_stderr --partial "crowdsecurity/syslog-logs: overwrite"
+    assert_stderr --partial "crowdsecurity/syslog-logs: updated"
+    assert_stderr --partial "Upgraded 1 parsers"
+    # this is used by the cron script to know if the hub was updated
+    assert_output --partial "updated crowdsecurity/syslog-logs"
 }

test/bats/20_hub_collections.bats

@@ -8,6 +8,8 @@ setup_file() {
     ./instance-data load
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
+    INDEX_PATH=$(config_get '.config_paths.index_path')
+    export INDEX_PATH
     CONFIG_DIR=$(config_get '.config_paths.config_dir')
     export CONFIG_DIR
 }
@@ -20,7 +22,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_purge_all
     hub_strip_index
 }
 
@@ -31,6 +32,8 @@ teardown() {
 #----------
 
 @test "cscli collections list" {
+    hub_purge_all
+
     # no items
     rune -0 cscli collections list
     assert_output --partial "COLLECTIONS"
@@ -62,7 +65,7 @@ teardown() {
 }
 
 @test "cscli collections list -a" {
-    expected=$(jq <"$HUB_DIR/.index.json" -r '.collections | length')
+    expected=$(jq <"$INDEX_PATH" -r '.collections | length')
 
     rune -0 cscli collections list -a
     rune -0 grep -c disabled <(output)
@@ -76,7 +79,17 @@ teardown() {
     rune -0 grep -vc 'name,status,version,description' <(output)
     assert_output "$expected"
 
-    # XXX: check alphabetical order in human, json, raw
+    # the list should be the same in all formats, and sorted (not case sensitive)
+
+    list_raw=$(cscli collections list -o raw -a | tail -n +2 | cut -d, -f1)
+    list_human=$(cscli collections list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2)
+    list_json=$(cscli collections list -o json -a | jq -r '.collections[].name')
+
+    rune -0 sort -f <<<"$list_raw"
+    assert_output "$list_raw"
+
+    assert_equal "$list_raw" "$list_json"
+    assert_equal "$list_raw" "$list_human"
 }
 
 @test "cscli collections list [collection]..." {
@@ -117,7 +130,7 @@ teardown() {
     assert_output "2"
 }
 
-@test "cscli collections install [collection]..." {
+@test "cscli collections install" {
     rune -1 cscli collections install
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
 
@@ -145,8 +158,7 @@ teardown() {
     assert_output --partial 'installed: true'
 }
 
-@test "cscli collections install [collection]... (file location and download-only)" {
+@test "cscli collections install (file location and download-only)" {
-    # simple install
     rune -0 cscli collections install crowdsecurity/linux --download-only
     rune -0 cscli collections inspect crowdsecurity/linux --no-metrics
     assert_output --partial 'crowdsecurity/linux'
@@ -155,11 +167,34 @@ teardown() {
     assert_file_not_exists "$CONFIG_DIR/collections/linux.yaml"
 
     rune -0 cscli collections install crowdsecurity/linux
+    rune -0 cscli collections inspect crowdsecurity/linux --no-metrics
+    assert_output --partial 'installed: true'
     assert_file_exists "$CONFIG_DIR/collections/linux.yaml"
 }
 
+@test "cscli collections install --force (tainted)" {
+    rune -0 cscli collections install crowdsecurity/sshd
+    echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
 
-@test "cscli collections inspect [collection]..." {
+    rune -1 cscli collections install crowdsecurity/sshd
+    assert_stderr --partial "error while installing 'crowdsecurity/sshd': while enabling crowdsecurity/sshd: crowdsecurity/sshd is tainted, won't enable unless --force"
+
+    rune -0 cscli collections install crowdsecurity/sshd --force
+    assert_stderr --partial "crowdsecurity/sshd: overwrite"
+    assert_stderr --partial "Enabled crowdsecurity/sshd"
+}
+
+@test "cscli collections install --ignore (skip on errors)" {
+    rune -1 cscli collections install foo/bar crowdsecurity/sshd
+    assert_stderr --partial "can't find 'foo/bar' in collections"
+    refute_stderr --partial "Enabled collections: crowdsecurity/sshd"
+
+    rune -0 cscli collections install foo/bar crowdsecurity/sshd --ignore
+    assert_stderr --partial "can't find 'foo/bar' in collections"
+    assert_stderr --partial "Enabled collections: crowdsecurity/sshd"
+}
+
+@test "cscli collections inspect" {
     rune -1 cscli collections inspect
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
     # required for metrics
@@ -184,7 +219,6 @@ teardown() {
     # one item, json
     rune -0 cscli collections inspect crowdsecurity/sshd -o json
     rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output)
-    # XXX: .installed is missing -- not false
     assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false]'
 
     # one item, raw
@@ -217,25 +251,33 @@ teardown() {
     rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw
     assert_output --partial 'crowdsecurity/sshd'
     assert_output --partial 'crowdsecurity/smb'
-    run -1 grep -c 'Current metrics:' <(output)
+    rune -1 grep -c 'Current metrics:' <(output)
     assert_output "0"
 }
 
-@test "cscli collections remove [collection]..." {
+@test "cscli collections remove" {
     rune -1 cscli collections remove
     assert_stderr --partial "specify at least one collection to remove or '--all'"
     rune -1 cscli collections remove blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in collections"
 
-    rune -0 cscli collections remove crowdsecurity/sshd
-    assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no removal required'
-
     rune -0 cscli collections install crowdsecurity/sshd --download-only
     rune -0 cscli collections remove crowdsecurity/sshd
-    assert_stderr --partial 'removing crowdsecurity/sshd: already uninstalled'
+    assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove'
+
+    rune -0 cscli collections install crowdsecurity/sshd
+    rune -0 cscli collections remove crowdsecurity/sshd
+    assert_stderr --partial 'Removed crowdsecurity/sshd'
+
     rune -0 cscli collections remove crowdsecurity/sshd --purge
     assert_stderr --partial 'Removed source file [crowdsecurity/sshd]'
 
+    rune -0 cscli collections remove crowdsecurity/sshd
+    assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove'
+
+    rune -0 cscli collections remove crowdsecurity/sshd --purge
+    assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no need to remove'
+
     # install, then remove, check files
     rune -0 cscli collections install crowdsecurity/sshd
     assert_file_exists "$CONFIG_DIR/collections/sshd.yaml"
@@ -267,7 +309,7 @@ teardown() {
     assert_output "0"
 }
 
-@test "cscli collections remove [collections]... --force" {
+@test "cscli collections remove --force" {
     # remove a collections that belongs to a collection
     rune -0 cscli collections install crowdsecurity/linux
     rune -0 cscli collections remove crowdsecurity/sshd
@@ -275,11 +317,12 @@ teardown() {
     assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this collection"
 }
 
-@test "cscli collections upgrade [collection]..." {
+@test "cscli collections upgrade" {
     rune -1 cscli collections upgrade
     assert_stderr --partial "specify at least one collection to upgrade or '--all'"
     rune -1 cscli collections upgrade blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in collections"
+    rune -0 cscli collections remove crowdsecurity/exim --purge
     rune -1 cscli collections upgrade crowdsecurity/exim
     assert_stderr --partial "can't upgrade crowdsecurity/exim: not installed"
     rune -0 cscli collections install crowdsecurity/exim --download-only
@@ -290,8 +333,8 @@ teardown() {
     sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
 
     # add version 0.0 to all collections
-    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
+    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
-    echo "$new_hub" >"$HUB_DIR/.index.json"
+    echo "$new_hub" >"$INDEX_PATH"
  
     rune -0 cscli collections install crowdsecurity/sshd
 

test/bats/20_hub_collections_dep.bats

@@ -6,8 +6,8 @@ set -u
 setup_file() {
     load "../lib/setup_file.sh"
     ./instance-data load
-    HUB_DIR=$(config_get '.config_paths.hub_dir')
+    INDEX_PATH=$(config_get '.config_paths.index_path')
-    export HUB_DIR
+    export INDEX_PATH
     CONFIG_DIR=$(config_get '.config_paths.config_dir')
     export CONFIG_DIR
 }
@@ -20,7 +20,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_purge_all
     hub_strip_index
 }
 
@@ -32,8 +31,8 @@ teardown() {
 
 @test "cscli collections (dependencies)" {
     # inject a dependency: smb requires sshd
-    hub_dep=$(jq <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}')
+    hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}')
-    echo "$hub_dep" >"$HUB_DIR/.index.json"
+    echo "$hub_dep" >"$INDEX_PATH"
 
     # verify that installing smb brings sshd
     rune -0 cscli collections install crowdsecurity/smb
@@ -63,7 +62,7 @@ teardown() {
 
     # and now smb is tainted!
     rune -0 cscli collections inspect crowdsecurity/smb -o json
-    rune -0 jq -e '.tainted//false==true' <(output)
+    rune -0 jq -e '.tainted==true' <(output)
     rune -0 cscli collections remove crowdsecurity/smb --force
 
     # empty
@@ -75,12 +74,41 @@ teardown() {
 
     # taint on sshd means smb is tainted as well
     rune -0 cscli collections inspect crowdsecurity/smb -o json
-    jq -e '.tainted//false==false' <(output)
+    rune -0 jq -e '.tainted==false' <(output)
     echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
     rune -0 cscli collections inspect crowdsecurity/smb -o json
-    jq -e '.tainted//false==true' <(output)
+    rune -0 jq -e '.tainted==true' <(output)
 
     # now we can't remove smb without --force
     rune -1 cscli collections remove crowdsecurity/smb
-    assert_stderr --partial "unable to disable crowdsecurity/smb: crowdsecurity/smb is tainted, use '--force' to overwrite"
+    assert_stderr --partial "crowdsecurity/smb is tainted, use '--force' to remove"
+}
+
+@test "cscli collections (dependencies II: the revenge)" {
+    rune -0 cscli collections install crowdsecurity/wireguard baudneo/gotify
+    rune -0 cscli collections remove crowdsecurity/wireguard
+    assert_stderr --partial "crowdsecurity/syslog-logs was not removed because it also belongs to baudneo/gotify"
+    rune -0 cscli collections inspect crowdsecurity/wireguard -o json
+    rune -0 jq -e '.installed==false' <(output)
+}
+
+@test "cscli collections (dependencies III: origins)" {
+    # it is perfectly fine to remove an item belonging to a collection that we are removing anyway
+
+    # inject a dependency: sshd requires the syslog-logs parsers, but linux does too
+    hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/sshd":{parsers:["crowdsecurity/syslog-logs"]}}}')
+    echo "$hub_dep" >"$INDEX_PATH"
+
+    # verify that installing sshd brings syslog-logs
+    rune -0 cscli collections install crowdsecurity/sshd
+    rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json
+    rune -0 jq -e '.installed==true' <(output)
+
+    rune -0 cscli collections install crowdsecurity/linux
+
+    # removing linux should remove syslog-logs even though sshd depends on it
+    rune -0 cscli collections remove crowdsecurity/linux
+    refute_stderr --partial "crowdsecurity/syslog-logs was not removed"
+    rune -0 cscli parsers list -o json
+    rune -0 jq -e '.parsers | length == 0' <(output)
 }

test/bats/20_hub_items.bats

@@ -8,6 +8,8 @@ setup_file() {
     ./instance-data load
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
+    INDEX_PATH=$(config_get '.config_paths.index_path')
+    export INDEX_PATH
     CONFIG_DIR=$(config_get '.config_paths.config_dir')
     export CONFIG_DIR
 }
@@ -20,7 +22,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_purge_all
     hub_strip_index
 }
 
@@ -41,10 +42,10 @@ teardown() {
     # in a lexical vs semver sort. CrowdSec should report the latest version
 
     new_hub=$( \
-        jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" \
+        jq --arg DIGEST "$sha256_empty" <"$INDEX_PATH" \
         '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2":{"digest":$DIGEST, "deprecated": false}, "1.10": {"digest":$DIGEST, "deprecated": false}}}}}' \
     )
-    echo "$new_hub" >"$HUB_DIR/.index.json"
+    echo "$new_hub" >"$INDEX_PATH"
  
     rune -0 cscli collections install crowdsecurity/sshd
 
@@ -57,16 +58,39 @@ teardown() {
 }
 
 @test "hub index with invalid (non semver) version numbers" {
+    rune -0 cscli collections remove crowdsecurity/sshd --purge
+
     new_hub=$( \
-        jq <"$HUB_DIR/.index.json" \
+        jq <"$INDEX_PATH" \
         '. * {collections:{"crowdsecurity/sshd":{"versions":{"1.2.3.4":{"digest":"foo", "deprecated": false}}}}}' \
     )
-    echo "$new_hub" >"$HUB_DIR/.index.json"
+    echo "$new_hub" >"$INDEX_PATH"
  
     rune -0 cscli collections install crowdsecurity/sshd
-
+    rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics -o json
-    rune -1 cscli collections inspect crowdsecurity/sshd --no-metrics
     # XXX: we are on the verbose side here...
-    assert_stderr --partial "failed to read Hub index: failed to sync items: failed to scan $CONFIG_DIR: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version"
+    rune -0 jq -r ".msg" <(stderr)
+    assert_output --regexp "failed to read Hub index: failed to sync items: failed to scan .*: while syncing collections sshd.yaml: 1.2.3.4: Invalid Semantic Version. Run 'sudo cscli hub update' to download the index again"
 }
 
+@test "removing or purging an item already removed by hand" {
+    rune -0 cscli parsers install crowdsecurity/syslog-logs
+    rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json
+    rune -0 jq -r '.local_path' <(output)
+    rune -0 rm "$(output)"
+
+    rune -0 cscli parsers remove crowdsecurity/syslog-logs --debug
+    assert_stderr --partial "removing crowdsecurity/syslog-logs: not installed -- no need to remove"
+
+    rune -0 cscli parsers inspect crowdsecurity/syslog-logs -o json
+    rune -0 jq -r '.path' <(output)
+    rune -0 rm "$HUB_DIR/$(output)"
+
+    rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge
+    assert_stderr --partial "removing crowdsecurity/syslog-logs: not downloaded -- no need to remove"
+
+    rune -0 cscli parsers remove crowdsecurity/linux --all --error --purge --force
+    rune -0 cscli collections remove crowdsecurity/linux --all --error --purge --force
+    refute_output
+    refute_stderr
+}

test/bats/20_hub_parsers.bats

@@ -8,6 +8,8 @@ setup_file() {
     ./instance-data load
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
+    INDEX_PATH=$(config_get '.config_paths.index_path')
+    export INDEX_PATH
     CONFIG_DIR=$(config_get '.config_paths.config_dir')
     export CONFIG_DIR
 }
@@ -20,7 +22,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_purge_all
     hub_strip_index
 }
 
@@ -31,6 +32,8 @@ teardown() {
 #----------
 
 @test "cscli parsers list" {
+    hub_purge_all
+
     # no items
     rune -0 cscli parsers list
     assert_output --partial "PARSERS"
@@ -62,7 +65,7 @@ teardown() {
 }
 
 @test "cscli parsers list -a" {
-    expected=$(jq <"$HUB_DIR/.index.json" -r '.parsers | length')
+    expected=$(jq <"$INDEX_PATH" -r '.parsers | length')
 
     rune -0 cscli parsers list -a
     rune -0 grep -c disabled <(output)
@@ -76,7 +79,17 @@ teardown() {
     rune -0 grep -vc 'name,status,version,description' <(output)
     assert_output "$expected"
 
-    # XXX: check alphabetical order in human, json, raw
+    # the list should be the same in all formats, and sorted (not case sensitive)
+
+    list_raw=$(cscli parsers list -o raw -a | tail -n +2 | cut -d, -f1)
+    list_human=$(cscli parsers list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2)
+    list_json=$(cscli parsers list -o json -a | jq -r '.parsers[].name')
+
+    rune -0 sort -f <<<"$list_raw"
+    assert_output "$list_raw"
+
+    assert_equal "$list_raw" "$list_json"
+    assert_equal "$list_raw" "$list_human"
 }
 
 @test "cscli parsers list [parser]..." {
@@ -117,7 +130,7 @@ teardown() {
     assert_output "3"
 }
 
-@test "cscli parsers install [parser]..." {
+@test "cscli parsers install" {
     rune -1 cscli parsers install
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
 
@@ -145,8 +158,7 @@ teardown() {
     assert_output --partial 'installed: true'
 }
 
-@test "cscli parsers install [parser]... (file location and download-only)" {
+@test "cscli parsers install (file location and download-only)" {
-    # simple install
     rune -0 cscli parsers install crowdsecurity/whitelists --download-only
     rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics
     assert_output --partial 'crowdsecurity/whitelists'
@@ -155,13 +167,34 @@ teardown() {
     assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
 
     rune -0 cscli parsers install crowdsecurity/whitelists
+    rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics
+    assert_output --partial 'installed: true'
     assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
 }
 
-# XXX: test install with --force
+@test "cscli parsers install --force (tainted)" {
-# XXX: test install with --ignore
+    rune -0 cscli parsers install crowdsecurity/whitelists
+    echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
 
-@test "cscli parsers inspect [parser]..." {
+    rune -1 cscli parsers install crowdsecurity/whitelists
+    assert_stderr --partial "error while installing 'crowdsecurity/whitelists': while enabling crowdsecurity/whitelists: crowdsecurity/whitelists is tainted, won't enable unless --force"
+
+    rune -0 cscli parsers install crowdsecurity/whitelists --force
+    assert_stderr --partial "crowdsecurity/whitelists: overwrite"
+    assert_stderr --partial "Enabled crowdsecurity/whitelists"
+}
+
+@test "cscli parsers install --ignore (skip on errors)" {
+    rune -1 cscli parsers install foo/bar crowdsecurity/whitelists
+    assert_stderr --partial "can't find 'foo/bar' in parsers"
+    refute_stderr --partial "Enabled parsers: crowdsecurity/whitelists"
+
+    rune -0 cscli parsers install foo/bar crowdsecurity/whitelists --ignore
+    assert_stderr --partial "can't find 'foo/bar' in parsers"
+    assert_stderr --partial "Enabled parsers: crowdsecurity/whitelists"
+}
+
+@test "cscli parsers inspect" {
     rune -1 cscli parsers inspect
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
     # required for metrics
@@ -192,8 +225,8 @@ teardown() {
     # one item, raw
     rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw
     assert_line 'type: parsers'
-    assert_line 'stage: s01-parse'
     assert_line 'name: crowdsecurity/sshd-logs'
+    assert_line 'stage: s01-parse'
     assert_line 'author: crowdsecurity'
     assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
     assert_line 'installed: false'
@@ -220,25 +253,33 @@ teardown() {
     rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw
     assert_output --partial 'crowdsecurity/sshd-logs'
     assert_output --partial 'crowdsecurity/whitelists'
-    run -1 grep -c 'Current metrics:' <(output)
+    rune -1 grep -c 'Current metrics:' <(output)
     assert_output "0"
 }
 
-@test "foo cscli parsers remove [parser]..." {
+@test "cscli parsers remove" {
     rune -1 cscli parsers remove
     assert_stderr --partial "specify at least one parser to remove or '--all'"
     rune -1 cscli parsers remove blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in parsers"
 
-    rune -0 cscli parsers remove crowdsecurity/whitelists
-    assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no removal required'
-
     rune -0 cscli parsers install crowdsecurity/whitelists --download-only
     rune -0 cscli parsers remove crowdsecurity/whitelists
-    assert_stderr --partial 'removing crowdsecurity/whitelists: already uninstalled'
+    assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove"
+
+    rune -0 cscli parsers install crowdsecurity/whitelists
+    rune -0 cscli parsers remove crowdsecurity/whitelists
+    assert_stderr --partial "Removed crowdsecurity/whitelists"
+
     rune -0 cscli parsers remove crowdsecurity/whitelists --purge
     assert_stderr --partial 'Removed source file [crowdsecurity/whitelists]'
 
+    rune -0 cscli parsers remove crowdsecurity/whitelists
+    assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove"
+
+    rune -0 cscli parsers remove crowdsecurity/whitelists --purge
+    assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no need to remove'
+
     # install, then remove, check files
     rune -0 cscli parsers install crowdsecurity/whitelists
     assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
@@ -270,19 +311,20 @@ teardown() {
     assert_output "0"
 }
 
-@test "cscli parsers remove [parser]... --force" {
+@test "cscli parsers remove --force" {
     # remove a parser that belongs to a collection
-    rune -0 cscli collections install crowdsecurity/linux
+    rune -0 cscli collections install crowdsecurity/sshd
     rune -0 cscli parsers remove crowdsecurity/sshd-logs
     assert_stderr --partial "crowdsecurity/sshd-logs belongs to collections: [crowdsecurity/sshd]"
     assert_stderr --partial "Run 'sudo cscli parsers remove crowdsecurity/sshd-logs --force' if you want to force remove this parser"
 }
 
-@test "cscli parsers upgrade [parser]..." {
+@test "cscli parsers upgrade" {
     rune -1 cscli parsers upgrade
     assert_stderr --partial "specify at least one parser to upgrade or '--all'"
     rune -1 cscli parsers upgrade blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in parsers"
+    rune -0 cscli parsers remove crowdsecurity/pam-logs --purge
     rune -1 cscli parsers upgrade crowdsecurity/pam-logs
     assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: not installed"
     rune -0 cscli parsers install crowdsecurity/pam-logs --download-only
@@ -293,8 +335,8 @@ teardown() {
     sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
 
     # add version 0.0 to all parsers
-    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
+    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
-    echo "$new_hub" >"$HUB_DIR/.index.json"
+    echo "$new_hub" >"$INDEX_PATH"
  
     rune -0 cscli parsers install crowdsecurity/whitelists
 

test/bats/20_hub_postoverflows.bats

@@ -8,6 +8,8 @@ setup_file() {
     ./instance-data load
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
+    INDEX_PATH=$(config_get '.config_paths.index_path')
+    export INDEX_PATH
     CONFIG_DIR=$(config_get '.config_paths.config_dir')
     export CONFIG_DIR
 }
@@ -20,7 +22,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_purge_all
     hub_strip_index
 }
 
@@ -31,6 +32,8 @@ teardown() {
 #----------
 
 @test "cscli postoverflows list" {
+    hub_purge_all
+
     # no items
     rune -0 cscli postoverflows list
     assert_output --partial "POSTOVERFLOWS"
@@ -62,7 +65,7 @@ teardown() {
 }
 
 @test "cscli postoverflows list -a" {
-    expected=$(jq <"$HUB_DIR/.index.json" -r '.postoverflows | length')
+    expected=$(jq <"$INDEX_PATH" -r '.postoverflows | length')
 
     rune -0 cscli postoverflows list -a
     rune -0 grep -c disabled <(output)
@@ -76,15 +79,24 @@ teardown() {
     rune -0 grep -vc 'name,status,version,description' <(output)
     assert_output "$expected"
 
-    # XXX: check alphabetical order in human, json, raw
+    # the list should be the same in all formats, and sorted (not case sensitive)
+
+    list_raw=$(cscli postoverflows list -o raw -a | tail -n +2 | cut -d, -f1)
+    list_human=$(cscli postoverflows list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2)
+    list_json=$(cscli postoverflows list -o json -a | jq -r '.postoverflows[].name')
+
+    rune -0 sort -f <<<"$list_raw"
+    assert_output "$list_raw"
+
+    assert_equal "$list_raw" "$list_json"
+    assert_equal "$list_raw" "$list_human"
 }
 
-@test "cscli postoverflows list [scenario]..." {
+@test "cscli postoverflows list [postoverflow]..." {
     # non-existent
     rune -1 cscli postoverflows install foo/bar
     assert_stderr --partial "can't find 'foo/bar' in postoverflows"
 
-
     # not installed
     rune -0 cscli postoverflows list crowdsecurity/rdns
     assert_output --regexp 'crowdsecurity/rdns.*disabled'
@@ -118,7 +130,7 @@ teardown() {
     assert_output "3"
 }
 
-@test "cscli postoverflows install [scenario]..." {
+@test "cscli postoverflows install" {
     rune -1 cscli postoverflows install
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
 
@@ -146,8 +158,7 @@ teardown() {
     assert_output --partial 'installed: true'
 }
 
-@test "cscli postoverflows install [postoverflow]... (file location and download-only)" {
+@test "cscli postoverflows install (file location and download-only)" {
-    # simple install
     rune -0 cscli postoverflows install crowdsecurity/rdns --download-only
     rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics
     assert_output --partial 'crowdsecurity/rdns'
@@ -156,13 +167,34 @@ teardown() {
     assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
 
     rune -0 cscli postoverflows install crowdsecurity/rdns
+    rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics
+    assert_output --partial 'installed: true'
     assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
 }
 
-# XXX: test install with --force
+@test "cscli postoverflows install --force (tainted)" {
-# XXX: test install with --ignore
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+    echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
 
-@test "cscli postoverflows inspect [scenario]..." {
+    rune -1 cscli postoverflows install crowdsecurity/rdns
+    assert_stderr --partial "error while installing 'crowdsecurity/rdns': while enabling crowdsecurity/rdns: crowdsecurity/rdns is tainted, won't enable unless --force"
+
+    rune -0 cscli postoverflows install crowdsecurity/rdns --force
+    assert_stderr --partial "crowdsecurity/rdns: overwrite"
+    assert_stderr --partial "Enabled crowdsecurity/rdns"
+}
+
+@test "cscli postoverflow install --ignore (skip on errors)" {
+    rune -1 cscli postoverflows install foo/bar crowdsecurity/rdns
+    assert_stderr --partial "can't find 'foo/bar' in postoverflows"
+    refute_stderr --partial "Enabled postoverflows: crowdsecurity/rdns"
+
+    rune -0 cscli postoverflows install foo/bar crowdsecurity/rdns --ignore
+    assert_stderr --partial "can't find 'foo/bar' in postoverflows"
+    assert_stderr --partial "Enabled postoverflows: crowdsecurity/rdns"
+}
+
+@test "cscli postoverflows inspect" {
     rune -1 cscli postoverflows inspect
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
     # required for metrics
@@ -188,14 +220,13 @@ teardown() {
     # one item, json
     rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
     rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output)
-    # XXX: .installed is missing -- not false
     assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false]'
 
     # one item, raw
     rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw
     assert_line 'type: postoverflows'
-    assert_line 'stage: s00-enrich'
     assert_line 'name: crowdsecurity/rdns'
+    assert_line 'stage: s00-enrich'
     assert_line 'author: crowdsecurity'
     assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
     assert_line 'installed: false'
@@ -226,21 +257,29 @@ teardown() {
     assert_output "0"
 }
 
-@test "cscli postoverflows remove [postoverflow]..." {
+@test "cscli postoverflows remove" {
     rune -1 cscli postoverflows remove
     assert_stderr --partial "specify at least one postoverflow to remove or '--all'"
     rune -1 cscli postoverflows remove blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows"
 
-    rune -0 cscli postoverflows remove crowdsecurity/rdns
-    assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no removal required'
-
     rune -0 cscli postoverflows install crowdsecurity/rdns --download-only
     rune -0 cscli postoverflows remove crowdsecurity/rdns
-    assert_stderr --partial 'removing crowdsecurity/rdns: already uninstalled'
+    assert_stderr --partial "removing crowdsecurity/rdns: not installed -- no need to remove"
+
+    rune -0 cscli postoverflows install crowdsecurity/rdns
+    rune -0 cscli postoverflows remove crowdsecurity/rdns
+    assert_stderr --partial 'Removed crowdsecurity/rdns'
+
     rune -0 cscli postoverflows remove crowdsecurity/rdns --purge
     assert_stderr --partial 'Removed source file [crowdsecurity/rdns]'
 
+    rune -0 cscli postoverflows remove crowdsecurity/rdns
+    assert_stderr --partial 'removing crowdsecurity/rdns: not installed -- no need to remove'
+
+    rune -0 cscli postoverflows remove crowdsecurity/rdns --purge
+    assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no need to remove'
+
     # install, then remove, check files
     rune -0 cscli postoverflows install crowdsecurity/rdns
     assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
@@ -272,7 +311,7 @@ teardown() {
     assert_output "0"
 }
 
-@test "cscli postoverflows remove [postoverflow]... --force" {
+@test "cscli postoverflows remove --force" {
     # remove a postoverflow that belongs to a collection
     rune -0 cscli collections install crowdsecurity/auditd
     rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process
@@ -280,11 +319,12 @@ teardown() {
     assert_stderr --partial "Run 'sudo cscli postoverflows remove crowdsecurity/auditd-whitelisted-process --force' if you want to force remove this postoverflow"
 }
 
-@test "cscli postoverflows upgrade [postoverflow]..." {
+@test "cscli postoverflows upgrade" {
     rune -1 cscli postoverflows upgrade
     assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'"
     rune -1 cscli postoverflows upgrade blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows"
+    rune -0 cscli postoverflows remove crowdsecurity/discord-crawler-whitelist --purge
     rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist
     assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: not installed"
     rune -0 cscli postoverflows install crowdsecurity/discord-crawler-whitelist --download-only
@@ -295,8 +335,8 @@ teardown() {
     sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
 
     # add version 0.0 to all postoverflows
-    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
+    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
-    echo "$new_hub" >"$HUB_DIR/.index.json"
+    echo "$new_hub" >"$INDEX_PATH"
  
     rune -0 cscli postoverflows install crowdsecurity/rdns
 

test/bats/20_hub_scenarios.bats

@@ -8,6 +8,8 @@ setup_file() {
     ./instance-data load
     HUB_DIR=$(config_get '.config_paths.hub_dir')
     export HUB_DIR
+    INDEX_PATH=$(config_get '.config_paths.index_path')
+    export INDEX_PATH
     CONFIG_DIR=$(config_get '.config_paths.config_dir')
     export CONFIG_DIR
 }
@@ -20,7 +22,6 @@ setup() {
     load "../lib/setup.sh"
     load "../lib/bats-file/load.bash"
     ./instance-data load
-    hub_purge_all
     hub_strip_index
 }
 
@@ -31,6 +32,8 @@ teardown() {
 #----------
 
 @test "cscli scenarios list" {
+    hub_purge_all
+
     # no items
     rune -0 cscli scenarios list
     assert_output --partial "SCENARIOS"
@@ -62,7 +65,7 @@ teardown() {
 }
 
 @test "cscli scenarios list -a" {
-    expected=$(jq <"$HUB_DIR/.index.json" -r '.scenarios | length')
+    expected=$(jq <"$INDEX_PATH" -r '.scenarios | length')
 
     rune -0 cscli scenarios list -a
     rune -0 grep -c disabled <(output)
@@ -76,7 +79,17 @@ teardown() {
     rune -0 grep -vc 'name,status,version,description' <(output)
     assert_output "$expected"
 
-    # XXX: check alphabetical order in human, json, raw
+    # the list should be the same in all formats, and sorted (not case sensitive)
+
+    list_raw=$(cscli scenarios list -o raw -a | tail -n +2 | cut -d, -f1)
+    list_human=$(cscli scenarios list -o human -a | tail -n +6 | head -n -1 | cut -d' ' -f2)
+    list_json=$(cscli scenarios list -o json -a | jq -r '.scenarios[].name')
+
+    rune -0 sort -f <<<"$list_raw"
+    assert_output "$list_raw"
+
+    assert_equal "$list_raw" "$list_json"
+    assert_equal "$list_raw" "$list_human"
 }
 
 @test "cscli scenarios list [scenario]..." {
@@ -117,7 +130,7 @@ teardown() {
     assert_output "3"
 }
 
-@test "cscli scenarios install [scenario]..." {
+@test "cscli scenarios install" {
     rune -1 cscli scenarios install
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
 
@@ -145,7 +158,7 @@ teardown() {
     assert_output --partial 'installed: true'
 }
 
-@test "cscli scenarios install [scenario]... (file location and download-only)" {
+@test "cscli scenarios install (file location and download-only)" {
     # simple install
     rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only
     rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics
@@ -155,14 +168,34 @@ teardown() {
     assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
 
     rune -0 cscli scenarios install crowdsecurity/ssh-bf
+    rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics
+    assert_output --partial 'installed: true'
     assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
 }
 
-# XXX: test install with --force
+@test "cscli scenarios install --force (tainted)" {
-# XXX: test install with --ignore
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
+    echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
 
+    rune -1 cscli scenarios install crowdsecurity/ssh-bf
+    assert_stderr --partial "error while installing 'crowdsecurity/ssh-bf': while enabling crowdsecurity/ssh-bf: crowdsecurity/ssh-bf is tainted, won't enable unless --force"
 
-@test "cscli scenarios inspect [scenario]..." {
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf --force
+    assert_stderr --partial "crowdsecurity/ssh-bf: overwrite"
+    assert_stderr --partial "Enabled crowdsecurity/ssh-bf"
+}
+
+@test "cscli scenarios install --ignore (skip on errors)" {
+    rune -1 cscli scenarios install foo/bar crowdsecurity/ssh-bf
+    assert_stderr --partial "can't find 'foo/bar' in scenarios"
+    refute_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf"
+
+    rune -0 cscli scenarios install foo/bar crowdsecurity/ssh-bf --ignore
+    assert_stderr --partial "can't find 'foo/bar' in scenarios"
+    assert_stderr --partial "Enabled scenarios: crowdsecurity/ssh-bf"
+}
+
+@test "cscli scenarios inspect" {
     rune -1 cscli scenarios inspect
     assert_stderr --partial 'requires at least 1 arg(s), only received 0'
     # required for metrics
@@ -223,21 +256,29 @@ teardown() {
     assert_output "0"
 }
 
-@test "cscli scenarios remove [scenario]..." {
+@test "cscli scenarios remove" {
     rune -1 cscli scenarios remove
     assert_stderr --partial "specify at least one scenario to remove or '--all'"
     rune -1 cscli scenarios remove blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios"
 
-    rune -0 cscli scenarios remove crowdsecurity/ssh-bf
-    assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no removal required'
-
     rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only
     rune -0 cscli scenarios remove crowdsecurity/ssh-bf
-    assert_stderr --partial 'removing crowdsecurity/ssh-bf: already uninstalled'
+    assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove"
+
+    rune -0 cscli scenarios install crowdsecurity/ssh-bf
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf
+    assert_stderr --partial "Removed crowdsecurity/ssh-bf"
+
     rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge
     assert_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]'
 
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf
+    assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove"
+
+    rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge
+    assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no need to remove'
+
     # install, then remove, check files
     rune -0 cscli scenarios install crowdsecurity/ssh-bf
     assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml"
@@ -269,7 +310,7 @@ teardown() {
     assert_output "0"
 }
 
-@test "cscli scenarios remove [scenario]... --force" {
+@test "cscli scenarios remove --force" {
     # remove a scenario that belongs to a collection
     rune -0 cscli collections install crowdsecurity/sshd
     rune -0 cscli scenarios remove crowdsecurity/ssh-bf
@@ -277,11 +318,12 @@ teardown() {
     assert_stderr --partial "Run 'sudo cscli scenarios remove crowdsecurity/ssh-bf --force' if you want to force remove this scenario"
 }
 
-@test "cscli scenarios upgrade [scenario]..." {
+@test "cscli scenarios upgrade" {
     rune -1 cscli scenarios upgrade
     assert_stderr --partial "specify at least one scenario to upgrade or '--all'"
     rune -1 cscli scenarios upgrade blahblah/blahblah
     assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios"
+    rune -0 cscli scenarios remove crowdsecurity/vsftpd-bf --purge
     rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf
     assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: not installed"
     rune -0 cscli scenarios install crowdsecurity/vsftpd-bf --download-only
@@ -292,8 +334,8 @@ teardown() {
     sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
 
     # add version 0.0 to all scenarios
-    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
+    new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$INDEX_PATH" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
-    echo "$new_hub" >"$HUB_DIR/.index.json"
+    echo "$new_hub" >"$INDEX_PATH"
  
     rune -0 cscli scenarios install crowdsecurity/ssh-bf
 

test/bats/30_machines_tls.bats

@@ -78,15 +78,17 @@ teardown() {
 @test "missing key_file" {
     config_set '.api.server.tls.key_file=""'
 
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "missing TLS key file"
+        --err "missing TLS key file" \
+        "${CROWDSEC}"
 }
 
 @test "missing cert_file" {
     config_set '.api.server.tls.cert_file=""'
 
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "missing TLS cert file"
+        --err "missing TLS cert file" \
+        "${CROWDSEC}"
 }
 
 @test "invalid OU for agent" {

test/bats/40_cold-logs.bats

@@ -11,9 +11,13 @@ fake_log() {
 
 setup_file() {
     load "../lib/setup_file.sh"
-
     # we reset config and data, and only run the daemon once for all the tests in this file
     ./instance-data load
+
+    cscli collections install crowdsecurity/sshd --error
+    cscli parsers install crowdsecurity/syslog-logs --error
+    cscli parsers install crowdsecurity/dateparse-enrich --error
+
     ./instance-crowdsec start
 }
 

test/bats/40_live-ban.bats

@@ -13,6 +13,11 @@ setup_file() {
     load "../lib/setup_file.sh"
     # we reset config and data, but run the daemon only in the tests that need it
     ./instance-data load
+
+    cscli collections install crowdsecurity/sshd --error
+    cscli parsers install crowdsecurity/syslog-logs --error
+    cscli parsers install crowdsecurity/dateparse-enrich --error
+
 }
 
 teardown_file() {

test/bats/50_simulation.bats

@@ -12,6 +12,11 @@ fake_log() {
 setup_file() {
     load "../lib/setup_file.sh"
     ./instance-data load
+
+    cscli collections install crowdsecurity/sshd --error
+    cscli parsers install crowdsecurity/syslog-logs --error
+    cscli parsers install crowdsecurity/dateparse-enrich --error
+
     ./instance-crowdsec start
 }
 

test/bats/72_plugin_badconfig.bats

@@ -27,7 +27,7 @@ setup() {
 teardown() {
     ./instance-crowdsec stop
     rm -f "${PLUGIN_DIR}"/badname
-    chmod go-w "${PLUGIN_DIR}"/notification-http
+    chmod go-w "${PLUGIN_DIR}"/notification-http || true
 }
 
 #----------
@@ -35,36 +35,41 @@ teardown() {
 @test "misconfigured plugin, only user is empty" {
     config_set '.plugin_config.user="" | .plugin_config.group="nogroup"'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set"
+        --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" \
+        "${CROWDSEC}"
 }
 
 @test "misconfigured plugin, only group is empty" {
     config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set"
+        --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" \
+        "${CROWDSEC}"
 }
 
 @test "misconfigured plugin, user does not exist" {
     config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist"
+        --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" \
+        "${CROWDSEC}"
 }
 
 @test "misconfigured plugin, group does not exist" {
     config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist"
+        --err "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" \
+        "${CROWDSEC}"
 }
 
 @test "bad plugin name" {
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}"
+        --err "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" \
+        "${CROWDSEC}"
 }
 
 @test "duplicate notification config" {
@@ -75,48 +80,55 @@ teardown() {
     config_set "${PROFILES_PATH}" '.notifications=["slack_default"]'
     # the slack plugin may fail or not, but we just need the logs
     config_set '.common.log_media="stdout"'
-    rune timeout 2s "${CROWDSEC}"
+    rune wait-for \
-    assert_stderr --partial "notification 'email_default' is defined multiple times"
+        --err "notification 'email_default' is defined multiple times" \
+        "${CROWDSEC}"
 }
 
 @test "bad plugin permission (group writable)" {
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     chmod g+w "${PLUGIN_DIR}"/notification-http
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid"
+        --err "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" \
+        "${CROWDSEC}"
 }
 
 @test "bad plugin permission (world writable)" {
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
     chmod o+w "${PLUGIN_DIR}"/notification-http
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid"
+        --err "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" \
+        "${CROWDSEC}"
 }
 
 @test "config.yaml: missing .plugin_config section" {
     config_set 'del(.plugin_config)'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: plugins are enabled, but the plugin_config section is missing in the configuration"
+        --err "api server init: plugins are enabled, but the plugin_config section is missing in the configuration" \
+        "${CROWDSEC}"
 }
 
 @test "config.yaml: missing config_paths.notification_dir" {
     config_set 'del(.config_paths.notification_dir)'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: plugins are enabled, but config_paths.notification_dir is not defined"
+        --err "api server init: plugins are enabled, but config_paths.notification_dir is not defined" \
+        "${CROWDSEC}"
 }
 
 @test "config.yaml: missing config_paths.plugin_dir" {
     config_set 'del(.config_paths.plugin_dir)'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined"
+        --err "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" \
+        "${CROWDSEC}"
 }
 
 @test "unable to run plugin broker: while reading plugin config" {
     config_set '.config_paths.notification_dir="/this/path/does/not/exist"'
     config_set "${PROFILES_PATH}" '.notifications=["http_default"]'
-    rune -1 timeout 2s "${CROWDSEC}"
+    rune -0 wait-for \
-    assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory"
+        --err "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" \
+        "${CROWDSEC}"
 }

test/bats/81_alert_context.bats

@@ -20,6 +20,9 @@ teardown_file() {
 setup() {
     load "../lib/setup.sh"
     ./instance-data load
+    cscli collections install crowdsecurity/sshd --error
+    cscli parsers install crowdsecurity/syslog-logs --error
+    cscli parsers install crowdsecurity/dateparse-enrich --error
 }
 
 teardown() {

test/bats/testdata/explain/explain-log.txt

@@ -2,15 +2,10 @@ line: Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authenticatio
 	├ s00-raw
 	|	└ 🟢 crowdsecurity/syslog-logs (+12 ~9)
 	├ s01-parse
-	|	└ 🟢 crowdsecurity/sshd-logs (+8 ~1)
+	|	└ 🟢 crowdsecurity/sshd-logs (+8)
-	├ s02-enrich
-	|	├ 🟢 crowdsecurity/dateparse-enrich (+2 ~2)
-	|	├ 🟢 crowdsecurity/geoip-enrich (+10)
-	|	└ 🟢 crowdsecurity/whitelists (unchanged)
 	├-------- parser success 🟢
 	├ Scenarios
 		├ 🟢 crowdsecurity/ssh-bf
 		├ 🟢 crowdsecurity/ssh-bf_user-enum
 		├ 🟢 crowdsecurity/ssh-slow-bf
 		└ 🟢 crowdsecurity/ssh-slow-bf_user-enum
-

test/bin/wait-for

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+import asyncio
+import argparse
+import os
+import re
+import signal
+import sys
+
+DEFAULT_TIMEOUT = 30
+
+# TODO: signal handler to terminate spawned process group when wait-for is killed
+# TODO: better return codes esp. when matches are found
+# TODO: multiple patterns (multiple out, err, both)
+# TODO: print unmatched patterns
+
+
+async def terminate(p):
+    # Terminate the process group (shell, crowdsec plugins)
+    try:
+        os.killpg(os.getpgid(p.pid), signal.SIGTERM)
+    except ProcessLookupError:
+        pass
+
+
+async def monitor(cmd, args, want_out, want_err, timeout):
+    """Monitor a process and terminate it if a pattern is matched in stdout or stderr.
+
+    Args:
+        cmd: The command to run.
+        args: A list of arguments to pass to the command.
+        stdout: A regular expression pattern to search for in stdout.
+        stderr: A regular expression pattern to search for in stderr.
+        timeout: The maximum number of seconds to wait for the process to terminate.
+
+    Returns:
+        The exit code of the process.
+    """
+
+    status = None
+
+    async def read_stream(p, stream, outstream, pattern):
+        nonlocal status
+        if stream is None:
+            return
+        while True:
+            line = await stream.readline()
+            if line:
+                line = line.decode('utf-8')
+                outstream.write(line)
+                if pattern and pattern.search(line):
+                    await terminate(process)
+                    # this is nasty.
+                    # if we timeout, we want to return a different exit code
+                    # in case of a match, so that the caller can tell
+                    # if the application was still running.
+                    # XXX: still not good for match found, but return code != 0
+                    if timeout != DEFAULT_TIMEOUT:
+                        status = 128
+                    else:
+                        status = 0
+                    break
+            else:
+                break
+
+    process = await asyncio.create_subprocess_exec(
+        cmd,
+        *args,
+        # capture stdout
+        stdout=asyncio.subprocess.PIPE,
+        # capture stderr
+        stderr=asyncio.subprocess.PIPE,
+        # disable buffering
+        bufsize=0,
+        # create a new process group
+        # (required to kill child processes when cmd is a shell)
+        preexec_fn=os.setsid)
+
+    out_regex = re.compile(want_out) if want_out else None
+    err_regex = re.compile(want_err) if want_err else None
+
+    # Apply a timeout
+    try:
+        await asyncio.wait_for(
+            asyncio.wait([
+                asyncio.create_task(process.wait()),
+                asyncio.create_task(read_stream(process, process.stdout, sys.stdout, out_regex)),
+                asyncio.create_task(read_stream(process, process.stderr, sys.stderr, err_regex))
+            ]), timeout)
+        if status is None:
+            status = process.returncode
+    except asyncio.TimeoutError:
+        await terminate(process)
+        status = 241
+
+    # Return the same exit code, stdout and stderr as the spawned process
+    return status
+
+
+async def main():
+    parser = argparse.ArgumentParser(
+        description='Monitor a process and terminate it if a pattern is matched in stdout or stderr.')
+    parser.add_argument('cmd', help='The command to run.')
+    parser.add_argument('args', nargs=argparse.REMAINDER, help='A list of arguments to pass to the command.')
+    parser.add_argument('--out', default='', help='A regular expression pattern to search for in stdout.')
+    parser.add_argument('--err', default='', help='A regular expression pattern to search for in stderr.')
+    parser.add_argument('--timeout', type=float, default=DEFAULT_TIMEOUT)
+    args = parser.parse_args()
+
+    exit_code = await monitor(args.cmd, args.args, args.out, args.err, args.timeout)
+
+    sys.exit(exit_code)
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

test/lib/config/config-global

@@ -38,6 +38,8 @@ DATA_DIR="${LOCAL_DIR}/${REL_DATA_DIR}"
 export DATA_DIR
 CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}"
 export CONFIG_DIR
+HUB_DIR="${CONFIG_DIR}/hub"
+export HUB_DIR
 
 if [[ $(uname) == "OpenBSD" ]]; then
     TAR=gtar
@@ -52,6 +54,51 @@ remove_init_data() {
 
 # we need a separate function for initializing config when testing package
 # because we want to test the configuration as well
+preload_hub_items() {
+    # pre-download everything but don't install anything
+    # each test can install what it needs
+
+    echo "Purging existing hub..."
+
+    "$CSCLI" parsers delete --all --error --purge --force
+    "$CSCLI" scenarios delete --all --error --purge --force
+    "$CSCLI" postoverflows delete --all --error --purge --force
+    "$CSCLI" collections delete --all --error --purge --force
+
+    echo "Pre-downloading hub content..."
+
+    #shellcheck disable=SC2046
+    "$CSCLI" collections install \
+        $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
+        --download-only \
+        --error
+
+    #shellcheck disable=SC2046
+    "$CSCLI" parsers install \
+        $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
+        --download-only \
+        --error
+
+    #shellcheck disable=SC2046
+    "$CSCLI" scenarios install \
+        $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
+        --download-only \
+        --error
+
+    #shellcheck disable=SC2046
+    "$CSCLI" postoverflows install \
+        $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
+        --download-only \
+        --error
+
+    # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
+    # so we have to delete the links manually, and leave the downloaded files in place
+
+    "$CSCLI" parsers delete --all --error
+    "$CSCLI" scenarios delete --all --error
+    "$CSCLI" postoverflows delete --all --error
+}
+
 make_init_data() {
     ./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
 
@@ -61,9 +108,11 @@ make_init_data() {
     # when installed packages are always using sqlite, so no need to regenerate
     # local credz for sqlite
 
+    preload_hub_items
+
     [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto
 
-    mkdir -p "${LOCAL_INIT_DIR}"
+    mkdir -p "$LOCAL_INIT_DIR"
 
     ./instance-db dump "${LOCAL_INIT_DIR}/database"
 

test/lib/config/config-local

@@ -101,6 +101,50 @@ config_generate() {
     ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml"
 }
 
+preload_hub_items() {
+    # pre-download everything but don't install anything
+    # each test can install what it needs
+
+    echo "Purging existing hub..."
+
+    "$CSCLI" parsers delete --all --error --purge --force
+    "$CSCLI" scenarios delete --all --error --purge --force
+    "$CSCLI" postoverflows delete --all --error --purge --force
+    "$CSCLI" collections delete --all --error --purge --force
+
+    echo "Pre-downloading hub content..."
+
+    #shellcheck disable=SC2046
+    "$CSCLI" collections install \
+        $("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
+        --download-only \
+        --error
+
+    #shellcheck disable=SC2046
+    "$CSCLI" parsers install \
+        $("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
+        --download-only \
+        --error
+
+    #shellcheck disable=SC2046
+    "$CSCLI" scenarios install \
+        $("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
+        --download-only \
+        --error
+
+    #shellcheck disable=SC2046
+    "$CSCLI" postoverflows install \
+        $("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
+        --download-only \
+        --error
+
+    # XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
+    # so we have to delete the links manually, and leave the downloaded files in place
+
+    "$CSCLI" parsers delete --all --error
+    "$CSCLI" scenarios delete --all --error
+    "$CSCLI" postoverflows delete --all --error
+}
 
 make_init_data() {
     ./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
@@ -118,9 +162,8 @@ make_init_data() {
 
     "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
     "$CSCLI" --warning hub update
-    "$CSCLI" --warning collections install crowdsecurity/linux
+
-    # the whitelists are installed by the deb & rpm packages, so we test with the same config
+    preload_hub_items
-    "$CSCLI" --warning parsers install crowdsecurity/whitelists
 
     mkdir -p "$LOCAL_INIT_DIR"
 

test/lib/setup_file.sh

@@ -20,6 +20,7 @@ eval "$(debug)"
 # Allow tests to use relative paths for helper scripts.
 # shellcheck disable=SC2164
 cd "${TEST_DIR}"
+export PATH="${TEST_DIR}/bin:${PATH}"
 
 # complain if there's a crowdsec running system-wide or leftover from a previous test
 ./bin/assert-crowdsec-not-running