Refact cscli item listing, tests (#2547)

* hub diet; taint tests
* cmd/crowdsec-cli: split utils.go, moved cwhub.GetHubStatusForItemType()
* cscli: refactor hub list commands, fix edge cases

cmd/crowdsec-cli/collections.go

@@ -254,6 +254,11 @@ func runCollectionsInspect(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
 	for _, name := range args {
 		if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil {
 			return err
@@ -292,8 +297,9 @@ func runCollectionsList(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// XXX: will happily ignore missing collections
-	ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
+	if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil {
+		return err
+	}
 
 	return nil
 }

cmd/crowdsec-cli/hub.go

@@ -50,7 +50,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := require.Hub(csConfig); err != nil {
+	if err = require.Hub(csConfig); err != nil {
 		return err
 	}
 
@@ -62,9 +62,12 @@ func runHubList(cmd *cobra.Command, args []string) error {
 
 	cwhub.DisplaySummary()
 
-	ListItems(color.Output, []string{
+	err = ListItems(color.Output, []string{
 		cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
-	}, args, true, false, all)
+	}, nil, true, false, all)
+	if err != nil {
+		return err
+	}
 
 	return nil
 }

cmd/crowdsec-cli/item_metrics.go

@@ -0,0 +1,245 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/trace"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func ShowMetrics(hubItem *cwhub.Item) {
+	switch hubItem.Type {
+	case cwhub.PARSERS:
+		metrics := GetParserMetric(prometheusURL, hubItem.Name)
+		parserMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.SCENARIOS:
+		metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
+		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.COLLECTIONS:
+		for _, item := range hubItem.Parsers {
+			metrics := GetParserMetric(prometheusURL, item)
+			parserMetricsTable(color.Output, item, metrics)
+		}
+		for _, item := range hubItem.Scenarios {
+			metrics := GetScenarioMetric(prometheusURL, item)
+			scenarioMetricsTable(color.Output, item, metrics)
+		}
+		for _, item := range hubItem.Collections {
+			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
+			if hubItem == nil {
+				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
+			}
+			ShowMetrics(hubItem)
+		}
+	default:
+		log.Errorf("item of type '%s' is unknown", hubItem.Type)
+	}
+}
+
+// GetParserMetric is a complete rip from prom2json
+func GetParserMetric(url string, itemName string) map[string]map[string]int {
+	stats := make(map[string]map[string]int)
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric %v", metric.Labels)
+			} else {
+				if srctype, ok := metric.Labels["type"]; ok {
+					source = srctype + ":" + source
+				}
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_reader_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+					stats[source]["parsed"] = 0
+					stats[source]["reads"] = 0
+					stats[source]["unparsed"] = 0
+					stats[source]["hits"] = 0
+				}
+				stats[source]["reads"] += ival
+			case "cs_parser_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			case "cs_node_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["hits"] += ival
+			case "cs_node_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_node_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetScenarioMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["instantiation"] = 0
+	stats["curr_count"] = 0
+	stats["overflow"] = 0
+	stats["pour"] = 0
+	stats["underflow"] = 0
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_bucket_created_total":
+				stats["instantiation"] += ival
+			case "cs_buckets":
+				stats["curr_count"] += ival
+			case "cs_bucket_overflowed_total":
+				stats["overflow"] += ival
+			case "cs_bucket_poured_total":
+				stats["pour"] += ival
+			case "cs_bucket_underflowed_total":
+				stats["underflow"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetPrometheusMetric(url string) []*prom2json.Family {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+
+	return result
+}
+
+type unit struct {
+	value  int64
+	symbol string
+}
+
+var ranges = []unit{
+	{value: 1e18, symbol: "E"},
+	{value: 1e15, symbol: "P"},
+	{value: 1e12, symbol: "T"},
+	{value: 1e9, symbol: "G"},
+	{value: 1e6, symbol: "M"},
+	{value: 1e3, symbol: "k"},
+	{value: 1, symbol: ""},
+}
+
+func formatNumber(num int) string {
+	goodUnit := unit{}
+	for _, u := range ranges {
+		if int64(num) >= u.value {
+			goodUnit = u
+			break
+		}
+	}
+
+	if goodUnit.value == 1 {
+		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
+	}
+
+	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
+	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
+}

cmd/crowdsec-cli/item_suggest.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/agext/levenshtein"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"slices"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+const MaxDistance = 7
+
+func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
+	errMsg := ""
+	if score < MaxDistance {
+		errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem)
+	} else {
+		errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType)
+	}
+	if ignoreErr {
+		log.Error(errMsg)
+	} else {
+		log.Fatalf(errMsg)
+	}
+}
+
+func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
+	allItems := make([]string, 0)
+	nearestScore := 100
+	nearestItem := &cwhub.Item{}
+	hubItems := cwhub.GetItemMap(itemType)
+	for _, item := range hubItems {
+		allItems = append(allItems, item.Name)
+	}
+
+	for _, s := range allItems {
+		d := levenshtein.Distance(itemName, s, nil)
+		if d < nearestScore {
+			nearestScore = d
+			nearestItem = cwhub.GetItem(itemType, s)
+		}
+	}
+	return nearestItem, nearestScore
+}
+
+func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	if err := require.Hub(csConfig); err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+	hubItems := cwhub.GetItemMap(itemType)
+	for _, item := range hubItems {
+		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
+			comp = append(comp, item.Name)
+		}
+	}
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}
+
+func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	if err := require.Hub(csConfig); err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	items, err := cwhub.GetInstalledItemsAsString(itemType)
+	if err != nil {
+		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	if toComplete != "" {
+		for _, item := range items {
+			if strings.Contains(item, toComplete) {
+				comp = append(comp, item)
+			}
+		}
+	} else {
+		comp = items
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}

cmd/crowdsec-cli/items.go

@@ -0,0 +1,174 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"slices"
+	"sort"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+
+func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) {
+	itemNames := cwhub.GetItemNames(itemType)
+
+	notExist := []string{}
+	if len(args) > 0 {
+		installedOnly = false
+		for _, arg := range args {
+			if !slices.Contains(itemNames, arg) {
+				notExist = append(notExist, arg)
+			}
+		}
+	}
+
+	if len(notExist) > 0 {
+		return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
+	}
+
+	if len(args) > 0 {
+		itemNames = args
+	}
+
+	if installedOnly {
+		installed := []string{}
+		for _, item := range itemNames {
+			if cwhub.GetItem(itemType, item).Installed {
+				installed = append(installed, item)
+			}
+		}
+		return installed, nil
+	}
+	return itemNames, nil
+}
+
+
+func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
+	var err error
+	items := make(map[string][]string)
+	for _, itemType := range itemTypes {
+		if items[itemType], err = selectItems(itemType, args, !all); err != nil {
+			return err
+		}
+	}
+		
+	if csConfig.Cscli.Output == "human" {
+		for _, itemType := range itemTypes {
+			listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType])
+		}
+	} else if csConfig.Cscli.Output == "json" {
+		type itemHubStatus struct {
+			Name         string `json:"name"`
+			LocalVersion string `json:"local_version"`
+			LocalPath    string `json:"local_path"`
+			Description  string `json:"description"`
+			UTF8Status   string `json:"utf8_status"`
+			Status       string `json:"status"`
+		}
+
+		hubStatus := make(map[string][]itemHubStatus)
+		for _, itemType := range itemTypes {
+			// empty slice in case there are no items of this type
+			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
+			for i, itemName := range items[itemType] {
+				item := cwhub.GetItem(itemType, itemName)
+				status, emo := item.Status()
+				hubStatus[itemType][i] = itemHubStatus{
+					Name:         item.Name,
+					LocalVersion: item.LocalVersion,
+					LocalPath:    item.LocalPath,
+					Description:  item.Description,
+					Status:       status,
+					UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
+				}
+			}
+			h := hubStatus[itemType]
+			sort.Slice(h, func(i, j int) bool { return h[i].Name < h[j].Name })
+		}
+		x, err := json.MarshalIndent(hubStatus, "", " ")
+		if err != nil {
+			log.Fatalf("failed to unmarshal")
+		}
+		out.Write(x)
+	} else if csConfig.Cscli.Output == "raw" {
+		csvwriter := csv.NewWriter(out)
+		if showHeader {
+			header := []string{"name", "status", "version", "description"}
+			if showType {
+				header = append(header, "type")
+			}
+			err := csvwriter.Write(header)
+			if err != nil {
+				log.Fatalf("failed to write header: %s", err)
+			}
+
+		}
+		for _, itemType := range itemTypes {
+			for _, itemName := range items[itemType] {
+				item := cwhub.GetItem(itemType, itemName)
+				status, _ := item.Status()
+				if item.LocalVersion == "" {
+					item.LocalVersion = "n/a"
+				}
+				row := []string{
+					item.Name,
+					status,
+					item.LocalVersion,
+					item.Description,
+				}
+				if showType {
+					row = append(row, itemType)
+				}
+				err := csvwriter.Write(row)
+				if err != nil {
+					log.Fatalf("failed to write raw output : %s", err)
+				}
+			}
+		}
+		csvwriter.Flush()
+	}
+	return nil
+}
+
+func InspectItem(name string, itemType string, noMetrics bool) error {
+	hubItem := cwhub.GetItem(itemType, name)
+	if hubItem == nil {
+		return fmt.Errorf("can't find '%s' in %s", name, itemType)
+	}
+
+	var (
+		b   []byte
+		err error
+	)
+
+	switch csConfig.Cscli.Output {
+	case "human", "raw":
+		b, err = yaml.Marshal(*hubItem)
+		if err != nil {
+			return fmt.Errorf("unable to marshal item: %s", err)
+		}
+	case "json":
+		b, err = json.MarshalIndent(*hubItem, "", " ")
+		if err != nil {
+			return fmt.Errorf("unable to marshal item: %s", err)
+		}
+	}
+
+	fmt.Printf("%s", string(b))
+
+	if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
+		return nil
+	}
+
+	fmt.Printf("\nCurrent metrics: \n")
+	ShowMetrics(hubItem)
+
+	return nil
+}

cmd/crowdsec-cli/parsers.go

@@ -279,8 +279,9 @@ func runParsersList(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// XXX: will happily ignore missing parsers
-	ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
+	if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil {
+		return err
+	}
 
 	return nil
 }

cmd/crowdsec-cli/postoverflows.go

@@ -280,8 +280,9 @@ func runPostOverflowsList(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// XXX: will happily ignore missing postoverflows
-	ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
+	if err = ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all); err != nil {
+		return err
+	}
 
 	return nil
 }

cmd/crowdsec-cli/scenarios.go

@@ -279,8 +279,9 @@ func runScenariosList(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// XXX: will happily ignore missing scenarios
-	ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
+	if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil {
+		return err
+	}
 
 	return nil
 }

cmd/crowdsec-cli/support.go

@@ -131,7 +131,9 @@ func collectOSInfo() ([]byte, error) {
 func collectHubItems(itemType string) []byte {
 	out := bytes.NewBuffer(nil)
 	log.Infof("Collecting %s list", itemType)
-	ListItems(out, []string{itemType}, []string{}, false, true, false)
+	if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
 	return out.Bytes()
 }
 

cmd/crowdsec-cli/utils.go

@@ -1,36 +1,17 @@
 package main
 
 import (
-	"encoding/csv"
-	"encoding/json"
 	"fmt"
-	"io"
-	"math"
 	"net"
-	"net/http"
-	"slices"
-	"strconv"
 	"strings"
-	"time"
 
-	"github.com/fatih/color"
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/agext/levenshtein"
-	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-const MaxDistance = 7
-
 func printHelp(cmd *cobra.Command) {
 	err := cmd.Help()
 	if err != nil {
@@ -38,189 +19,6 @@ func printHelp(cmd *cobra.Command) {
 	}
 }
 
-func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
-	errMsg := ""
-	if score < MaxDistance {
-		errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem)
-	} else {
-		errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType)
-	}
-	if ignoreErr {
-		log.Error(errMsg)
-	} else {
-		log.Fatalf(errMsg)
-	}
-}
-
-func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
-	allItems := make([]string, 0)
-	nearestScore := 100
-	nearestItem := &cwhub.Item{}
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		allItems = append(allItems, item.Name)
-	}
-
-	for _, s := range allItems {
-		d := levenshtein.Distance(itemName, s, nil)
-		if d < nearestScore {
-			nearestScore = d
-			nearestItem = cwhub.GetItem(itemType, s)
-		}
-	}
-	return nearestItem, nearestScore
-}
-
-func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
-			comp = append(comp, item.Name)
-		}
-	}
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	items, err := cwhub.GetInstalledItemsAsString(itemType)
-	if err != nil {
-		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-
-	if toComplete != "" {
-		for _, item := range items {
-			if strings.Contains(item, toComplete) {
-				comp = append(comp, item)
-			}
-		}
-	} else {
-		comp = items
-	}
-
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
-	var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
-
-	for _, itemType := range itemTypes {
-		itemName := ""
-		if len(args) == 1 {
-			itemName = args[0]
-		}
-		hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
-	}
-
-	if csConfig.Cscli.Output == "human" {
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
-		}
-	} else if csConfig.Cscli.Output == "json" {
-		x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
-		if err != nil {
-			log.Fatalf("failed to unmarshal")
-		}
-		out.Write(x)
-	} else if csConfig.Cscli.Output == "raw" {
-		csvwriter := csv.NewWriter(out)
-		if showHeader {
-			header := []string{"name", "status", "version", "description"}
-			if showType {
-				header = append(header, "type")
-			}
-			err := csvwriter.Write(header)
-			if err != nil {
-				log.Fatalf("failed to write header: %s", err)
-			}
-
-		}
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			for _, status := range statuses {
-				if status.LocalVersion == "" {
-					status.LocalVersion = "n/a"
-				}
-				row := []string{
-					status.Name,
-					status.Status,
-					status.LocalVersion,
-					status.Description,
-				}
-				if showType {
-					row = append(row, itemType)
-				}
-				err := csvwriter.Write(row)
-				if err != nil {
-					log.Fatalf("failed to write raw output : %s", err)
-				}
-			}
-		}
-		csvwriter.Flush()
-	}
-}
-
-func InspectItem(name string, itemType string, noMetrics bool) error {
-	hubItem := cwhub.GetItem(itemType, name)
-	if hubItem == nil {
-		return fmt.Errorf("can't find '%s' in %s", name, itemType)
-	}
-
-	var (
-		b []byte
-		err error
-	)
-
-	switch csConfig.Cscli.Output {
-	case "human", "raw":
-		b, err = yaml.Marshal(*hubItem)
-		if err != nil {
-			return fmt.Errorf("unable to marshal item: %s", err)
-		}
-	case "json":
-		b, err = json.MarshalIndent(*hubItem, "", " ")
-		if err != nil {
-			return fmt.Errorf("unable to marshal item: %s", err)
-		}
-	}
-
-	fmt.Printf("%s", string(b))
-
-	if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
-		return nil
-	}
-
-	fmt.Printf("\nCurrent metrics: \n")
-	ShowMetrics(hubItem)
-
-	return nil
-}
-
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
 
 	/*if a range is provided, change the scope*/
@@ -251,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 	return nil
 }
 
-func ShowMetrics(hubItem *cwhub.Item) {
-	switch hubItem.Type {
-	case cwhub.PARSERS:
-		metrics := GetParserMetric(hubItem.Name)
-		parserMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.SCENARIOS:
-		metrics := GetScenarioMetric(hubItem.Name)
-		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.COLLECTIONS:
-		for _, item := range hubItem.Parsers {
-			metrics := GetParserMetric(item)
-			parserMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Scenarios {
-			metrics := GetScenarioMetric(item)
-			scenarioMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Collections {
-			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
-			if hubItem == nil {
-				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
-			}
-			ShowMetrics(hubItem)
-		}
-	default:
-		log.Errorf("item of type '%s' is unknown", hubItem.Type)
-	}
-}
-
-// GetParserMetric is a complete rip from prom2json
-func GetParserMetric(itemName string) map[string]map[string]int {
-	stats := make(map[string]map[string]int)
-
-	result := GetPrometheusMetric()
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			source, ok := metric.Labels["source"]
-			if !ok {
-				log.Debugf("no source in Metric %v", metric.Labels)
-			} else {
-				if srctype, ok := metric.Labels["type"]; ok {
-					source = srctype + ":" + source
-				}
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_reader_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-					stats[source]["parsed"] = 0
-					stats[source]["reads"] = 0
-					stats[source]["unparsed"] = 0
-					stats[source]["hits"] = 0
-				}
-				stats[source]["reads"] += ival
-			case "cs_parser_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_parser_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			case "cs_node_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["hits"] += ival
-			case "cs_node_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_node_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetScenarioMetric(itemName string) map[string]int {
-	stats := make(map[string]int)
-
-	stats["instantiation"] = 0
-	stats["curr_count"] = 0
-	stats["overflow"] = 0
-	stats["pour"] = 0
-	stats["underflow"] = 0
-
-	result := GetPrometheusMetric()
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_bucket_created_total":
-				stats["instantiation"] += ival
-			case "cs_buckets":
-				stats["curr_count"] += ival
-			case "cs_bucket_overflowed_total":
-				stats["overflow"] += ival
-			case "cs_bucket_poured_total":
-				stats["pour"] += ival
-			case "cs_bucket_underflowed_total":
-				stats["underflow"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetPrometheusMetric() []*prom2json.Family {
-	mfChan := make(chan *dto.MetricFamily, 1024)
-
-	// Start with the DefaultTransport for sane defaults.
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	// Conservatively disable HTTP keep-alives as this program will only
-	// ever need a single HTTP request.
-	transport.DisableKeepAlives = true
-	// Timeout early if the server doesn't even return the headers.
-	transport.ResponseHeaderTimeout = time.Minute
-
-	go func() {
-		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
-		err := prom2json.FetchMetricFamilies(csConfig.Cscli.PrometheusUrl, mfChan, transport)
-		if err != nil {
-			log.Fatalf("failed to fetch prometheus metrics : %v", err)
-		}
-	}()
-
-	result := []*prom2json.Family{}
-	for mf := range mfChan {
-		result = append(result, prom2json.NewFamily(mf))
-	}
-	log.Debugf("Finished reading prometheus output, %d entries", len(result))
-
-	return result
-}
-
-type unit struct {
-	value  int64
-	symbol string
-}
-
-var ranges = []unit{
-	{value: 1e18, symbol: "E"},
-	{value: 1e15, symbol: "P"},
-	{value: 1e12, symbol: "T"},
-	{value: 1e9,  symbol: "G"},
-	{value: 1e6,  symbol: "M"},
-	{value: 1e3,  symbol: "k"},
-	{value: 1,    symbol: ""},
-}
-
-func formatNumber(num int) string {
-	goodUnit := unit{}
-	for _, u := range ranges {
-		if int64(num) >= u.value {
-			goodUnit = u
-			break
-		}
-	}
-
-	if goodUnit.value == 1 {
-		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
-	}
-
-	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
-	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
-}
-
 func getDBClient() (*database.Client, error) {
 	var err error
 	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
@@ -510,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
 	}
 
 	return slice
-
 }

cmd/crowdsec-cli/utils_table.go

@@ -10,14 +10,16 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
+func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) {
 	t := newLightTable(out)
 	t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	for _, status := range statuses {
-		t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
+	for itemName := range itemNames {
+		item := cwhub.GetItem(itemType, itemNames[itemName])
+		status, emo := item.Status()
+		t.AddRow(item.Name, fmt.Sprintf("%v  %s", emo, status), item.LocalVersion, item.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()

pkg/cwhub/cwhub.go

@@ -1,10 +1,13 @@
+// Package cwhub is responsible for installing and upgrading the local hub files.
+//
+// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
+// and managing the dependencies and taints.
 package cwhub
 
 import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 
 	"github.com/enescakir/emoji"
@@ -36,18 +39,12 @@ var (
 	hubIdx             map[string]map[string]Item
 )
 
+// ItemVersion is used to detect the version of a given item
+// by comparing the hash of each version to the local file.
+// If the item does not match any known version, it is considered tainted.
 type ItemVersion struct {
-	Digest     string `json:"digest,omitempty"` // meow
-	Deprecated bool   `json:"deprecated,omitempty"`
-}
-
-type ItemHubStatus struct {
-	Name         string `json:"name"`
-	LocalVersion string `json:"local_version"`
-	LocalPath    string `json:"local_path"`
-	Description  string `json:"description"`
-	UTF8Status   string `json:"utf8_status"`
-	Status       string `json:"status"`
+	Digest     string `json:"digest,omitempty"`     // meow
+	Deprecated bool   `json:"deprecated,omitempty"` // XXX: do we keep this?
 }
 
 // Item can be: parser, scenario, collection..
@@ -84,7 +81,7 @@ type Item struct {
 	Collections   []string `json:"collections,omitempty"   yaml:"collections,omitempty"`
 }
 
-func (i *Item) status() (string, emoji.Emoji) {
+func (i *Item) Status() (string, emoji.Emoji) {
 	status := "disabled"
 	ok := false
 
@@ -124,19 +121,6 @@ func (i *Item) status() (string, emoji.Emoji) {
 	return status, emo
 }
 
-func (i *Item) hubStatus() ItemHubStatus {
-	status, emo := i.status()
-
-	return ItemHubStatus{
-		Name:         i.Name,
-		LocalVersion: i.LocalVersion,
-		LocalPath:    i.LocalPath,
-		Description:  i.Description,
-		Status:       status,
-		UTF8Status:   fmt.Sprintf("%v  %s", emo, status),
-	}
-}
-
 // versionStatus: semver requires 'v' prefix
 func (i *Item) versionStatus() int {
 	return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
@@ -206,6 +190,23 @@ func GetItem(itemType string, itemName string) *Item {
 	return nil
 }
 
+// GetItemNames returns the list of item (full) names for a given type
+// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
+// The names can be used to retrieve the item with GetItem()
+func GetItemNames(itemType string) []string {
+	m := GetItemMap(itemType)
+	if m == nil {
+		return nil
+	}
+
+	names := make([]string, 0, len(m))
+	for k := range m {
+		names = append(names, k)
+	}
+
+	return names
+}
+
 func AddItem(itemType string, item Item) error {
 	for _, itype := range ItemTypes {
 		if itype == itemType {
@@ -257,32 +258,3 @@ func GetInstalledItemsAsString(itemType string) ([]string, error) {
 
 	return retStr, nil
 }
-
-// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
-func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
-	if _, ok := hubIdx[itemType]; !ok {
-		log.Errorf("type %s doesn't exist", itemType)
-
-		return nil
-	}
-
-	ret := make([]ItemHubStatus, 0)
-
-	// remember, you do it for the user :)
-	for _, item := range hubIdx[itemType] {
-		if name != "" && name != item.Name {
-			// user has requested a specific name
-			continue
-		}
-		// Only enabled items ?
-		if !all && !item.Installed {
-			continue
-		}
-		// Check the item status
-		ret = append(ret, item.hubStatus())
-	}
-
-	sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
-
-	return ret
-}

pkg/cwhub/cwhub_test.go

@@ -53,7 +53,7 @@ func TestItemStatus(t *testing.T) {
 		item.Local = false
 		item.Tainted = false
 
-		txt, _ := item.status()
+		txt, _ := item.Status()
 		require.Equal(t, "enabled,update-available", txt)
 
 		item.Installed = false
@@ -61,7 +61,7 @@ func TestItemStatus(t *testing.T) {
 		item.Local = true
 		item.Tainted = false
 
-		txt, _ = item.status()
+		txt, _ = item.Status()
 		require.Equal(t, "disabled,local", txt)
 	}
 
@@ -273,10 +273,8 @@ func TestInstallParser(t *testing.T) {
 	for _, it := range hubIdx[PARSERS] {
 		testInstallItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
-		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
 		testTaintItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
-		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
 		testUpdateItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
 		testDisableItem(cfg.Hub, t, it)
@@ -309,11 +307,6 @@ func TestInstallCollection(t *testing.T) {
 		testUpdateItem(cfg.Hub, t, it)
 		it = hubIdx[COLLECTIONS][it.Name]
 		testDisableItem(cfg.Hub, t, it)
-
-		it = hubIdx[COLLECTIONS][it.Name]
-		x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
-		log.Infof("%+v", x)
-
 		break
 	}
 }

test/bats/20_hub_parsers.bats

@@ -79,41 +79,42 @@ teardown() {
     assert_output "$expected"
 }
 
-
 @test "cscli parsers list [parser]..." {
+    # non-existent
+    rune -1 cscli parsers install foo/bar
+    assert_stderr --partial "can't find 'foo/bar' in parsers"
+
+    # not installed
+    rune -0 cscli parsers list crowdsecurity/whitelists
+    assert_output --regexp 'crowdsecurity/whitelists.*disabled'
+
+    # install two items
     rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth
 
-    # list one item
+    # list an installed item
     rune -0 cscli parsers list crowdsecurity/whitelists
-    assert_output --partial "crowdsecurity/whitelists"
+    assert_output --regexp "crowdsecurity/whitelists.*enabled"
     refute_output --partial "crowdsecurity/windows-auth"
 
-    # list multiple items
-    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth
+    # list multiple installed and non installed items
+    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs
     assert_output --partial "crowdsecurity/whitelists"
     assert_output --partial "crowdsecurity/windows-auth"
+    assert_output --partial "crowdsecurity/traefik-logs"
 
     rune -0 cscli parsers list crowdsecurity/whitelists -o json
     rune -0 jq '.parsers | length' <(output)
     assert_output "1"
-    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o json
+    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json
     rune -0 jq '.parsers | length' <(output)
-    assert_output "2"
+    assert_output "3"
 
     rune -0 cscli parsers list crowdsecurity/whitelists -o raw
     rune -0 grep -vc 'name,status,version,description' <(output)
     assert_output "1"
-    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth -o raw
+    rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw
     rune -0 grep -vc 'name,status,version,description' <(output)
-    assert_output "2"
-}
-
-@test "cscli parsers list [parser]... (not installed / not existing)" {
-    skip "not implemented yet"
-    # not installed
-    rune -1 cscli parsers list crowdsecurity/whitelists
-    # not existing
-    rune -1 cscli parsers list blahblah/blahblah
+    assert_output "3"
 }
 
 @test "cscli parsers install [parser]..." {

test/bats/20_hub_scenarios.bats

@@ -77,41 +77,42 @@ teardown() {
     assert_output "$expected"
 }
 
-
 @test "cscli scenarios list [scenario]..." {
+    # non-existent
+    rune -1 cscli scenario install foo/bar
+    assert_stderr --partial "can't find 'foo/bar' in scenarios"
+ 
+    # not installed
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf
+    assert_output --regexp 'crowdsecurity/ssh-bf.*disabled'
+
+    # install two items
     rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf
 
-    # list one item
+    # list an installed item
     rune -0 cscli scenarios list crowdsecurity/ssh-bf
-    assert_output --partial "crowdsecurity/ssh-bf"
+    assert_output --regexp "crowdsecurity/ssh-bf.*enabled"
     refute_output --partial "crowdsecurity/telnet-bf"
 
-    # list multiple items
-    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf
+    # list multiple installed and non installed items
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf
     assert_output --partial "crowdsecurity/ssh-bf"
     assert_output --partial "crowdsecurity/telnet-bf"
+    assert_output --partial "crowdsecurity/aws-bf"
 
     rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json
     rune -0 jq '.scenarios | length' <(output)
     assert_output "1"
-    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json
     rune -0 jq '.scenarios | length' <(output)
-    assert_output "2"
+    assert_output "3"
 
     rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw
     rune -0 grep -vc 'name,status,version,description' <(output)
     assert_output "1"
-    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw
+    rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw
     rune -0 grep -vc 'name,status,version,description' <(output)
-    assert_output "2"
-}
-
-@test "cscli scenarios list [scenario]... (not installed / not existing)" {
-    skip "not implemented yet"
-    # not installed
-    rune -1 cscli scenarios list crowdsecurity/ssh-bf
-    # not existing
-    rune -1 cscli scenarios list blahblah/blahblah
+    assert_output "3"
 }
 
 @test "cscli scenarios install [scenario]..." {