Fix json output of cscli hub list (#1143)

* Fix json output of cscli hub list
* Fix functional tests.

Signed-off-by: Shivam Sandbhor <shivam.sandbhor@gmail.com>

cmd/crowdsec-cli/collections.go

@@ -142,7 +142,7 @@ func NewCollectionsCmd() *cobra.Command {
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			ListItem(cwhub.COLLECTIONS, args, false, true)
+			ListItems([]string{cwhub.COLLECTIONS}, args, false, true)
 		},
 	}
 	cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")

cmd/crowdsec-cli/hub.go

@@ -56,14 +56,7 @@ cscli hub update # Download list of available configurations from the hub
 				log.Info(v)
 			}
 			cwhub.DisplaySummary()
-			log.Printf("PARSERS:")
-			ListItem(cwhub.PARSERS, args, true, true)
-			log.Printf("SCENARIOS:")
-			ListItem(cwhub.SCENARIOS, args, true, false)
-			log.Printf("COLLECTIONS:")
-			ListItem(cwhub.COLLECTIONS, args, true, false)
-			log.Printf("POSTOVERFLOWS:")
-			ListItem(cwhub.PARSERS_OVFLW, args, true, false)
+			ListItems([]string{cwhub.PARSERS, cwhub.COLLECTIONS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW}, args, true, false)
 		},
 	}
 	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")

cmd/crowdsec-cli/parsers.go

@@ -132,7 +132,7 @@ cscli parsers remove crowdsecurity/sshd-logs
 cscli parser list crowdsecurity/xxx`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			ListItem(cwhub.PARSERS, args, false, true)
+			ListItems([]string{cwhub.PARSERS}, args, false, true)
 		},
 	}
 	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")

cmd/crowdsec-cli/postoverflows.go

@@ -130,7 +130,7 @@ func NewPostOverflowsCmd() *cobra.Command {
 cscli postoverflows list crowdsecurity/xxx`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			ListItem(cwhub.PARSERS_OVFLW, args, false, true)
+			ListItems([]string{cwhub.PARSERS_OVFLW}, args, false, true)
 		},
 	}
 	cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")

cmd/crowdsec-cli/scenarios.go

@@ -132,7 +132,7 @@ cscli scenarios remove crowdsecurity/ssh-bf
 cscli scenarios list crowdsecurity/xxx`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			ListItem(cwhub.SCENARIOS, args, false, true)
+			ListItems([]string{cwhub.SCENARIOS}, args, false, true)
 		},
 	}
 	cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")

cmd/crowdsec-cli/utils.go

@@ -102,31 +102,35 @@ func setHubBranch() error {
 	return nil
 }
 
-func ListItem(itemType string, args []string, showType bool, showHeader bool) {
+func ListItems(itemTypes []string, args []string, showType bool, showHeader bool) {
 
-	var hubStatus []map[string]string
+	var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
 
-	if len(args) == 1 {
-		hubStatus = cwhub.HubStatus(itemType, args[0], all)
-	} else {
-		hubStatus = cwhub.HubStatus(itemType, "", all)
+	for _, itemType := range itemTypes {
+		if len(args) == 1 {
+			// This means that user requested a specific item by name
+			hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, args[0], all)
+		} else {
+			hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, "", all)
+		}
 	}
 
 	if csConfig.Cscli.Output == "human" {
-
-		table := tablewriter.NewWriter(os.Stdout)
-		table.SetCenterSeparator("")
-		table.SetColumnSeparator("")
-
-		table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
-		table.SetAlignment(tablewriter.ALIGN_LEFT)
-		table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
-		for _, v := range hubStatus {
-			table.Append([]string{v["name"], v["utf8_status"], v["local_version"], v["local_path"]})
+		for itemType, statuses := range hubStatusByItemType {
+			fmt.Println(strings.ToUpper(itemType))
+			table := tablewriter.NewWriter(os.Stdout)
+			table.SetCenterSeparator("")
+			table.SetColumnSeparator("")
+			table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
+			table.SetAlignment(tablewriter.ALIGN_LEFT)
+			table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
+			for _, status := range statuses {
+				table.Append([]string{status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath})
+			}
+			table.Render()
 		}
-		table.Render()
 	} else if csConfig.Cscli.Output == "json" {
-		x, err := json.MarshalIndent(hubStatus, "", " ")
+		x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
 		if err != nil {
 			log.Fatalf("failed to unmarshal")
 		}
@@ -147,22 +151,24 @@ func ListItem(itemType string, args []string, showType bool, showHeader bool) {
 			}
 
 		}
-		for _, v := range hubStatus {
-			if v["local_version"] == "" {
-				v["local_version"] = "n/a"
-			}
-			row := []string{
-				v["name"],
-				v["status"],
-				v["local_version"],
-				v["description"],
-			}
-			if showType {
-				row = append(row, itemType)
-			}
-			err := csvwriter.Write(row)
-			if err != nil {
-				log.Fatalf("failed to write raw output : %s", err)
+		for itemType, statuses := range hubStatusByItemType {
+			for _, status := range statuses {
+				if status.LocalVersion == "" {
+					status.LocalVersion = "n/a"
+				}
+				row := []string{
+					status.Name,
+					status.Status,
+					status.LocalVersion,
+					status.Description,
+				}
+				if showType {
+					row = append(row, itemType)
+				}
+				err := csvwriter.Write(row)
+				if err != nil {
+					log.Fatalf("failed to write raw output : %s", err)
+				}
 			}
 		}
 		csvwriter.Flush()

pkg/cwhub/cwhub.go

@@ -35,6 +35,15 @@ type ItemVersion struct {
 	Deprecated bool
 }
 
+type ItemHubStatus struct {
+	Name         string `json:"name"`
+	LocalVersion string `json:"local_version"`
+	LocalPath    string `json:"local_path"`
+	Description  string `json:"description"`
+	UTF8_Status  string `json:"utf8_status"`
+	Status       string `json:"status"`
+}
+
 //Item can be : parsed, scenario, collection
 type Item struct {
 	/*descriptive info*/
@@ -72,6 +81,27 @@ type Item struct {
 	Collections   []string `yaml:"collections,omitempty"`
 }
 
+func (i *Item) toHubStatus() ItemHubStatus {
+	hubStatus := ItemHubStatus{}
+	hubStatus.Name = i.Name
+	hubStatus.LocalVersion = i.LocalVersion
+	hubStatus.LocalPath = i.LocalPath
+	hubStatus.Description = i.Description
+
+	status, ok, warning, managed := ItemStatus(*i)
+	hubStatus.Status = status
+	if !managed {
+		hubStatus.UTF8_Status = fmt.Sprintf("%v  %s", emoji.House, status)
+	} else if !i.Installed {
+		hubStatus.UTF8_Status = fmt.Sprintf("%v  %s", emoji.Prohibited, status)
+	} else if warning {
+		hubStatus.UTF8_Status = fmt.Sprintf("%v  %s", emoji.Warning, status)
+	} else if ok {
+		hubStatus.UTF8_Status = fmt.Sprintf("%v  %s", emoji.CheckMark, status)
+	}
+	return hubStatus
+}
+
 var skippedLocal = 0
 var skippedTainted = 0
 
@@ -240,18 +270,18 @@ func GetUpstreamInstalledScenarios() ([]Item, error) {
 }
 
 //Returns a list of entries for packages : name, status, local_path, local_version, utf8_status (fancy)
-func HubStatus(itemType string, name string, all bool) []map[string]string {
+func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
 	if _, ok := hubIdx[itemType]; !ok {
 		log.Errorf("type %s doesn't exist", itemType)
 
 		return nil
 	}
 
-	var ret []map[string]string
+	var ret = make([]ItemHubStatus, 0)
 	/*remember, you do it for the user :)*/
 	for _, item := range hubIdx[itemType] {
 		if name != "" && name != item.Name {
-			//user has required a specific name
+			//user has requested a specific name
 			continue
 		}
 		//Only enabled items ?
@@ -259,23 +289,7 @@ func HubStatus(itemType string, name string, all bool) []map[string]string {
 			continue
 		}
 		//Check the item status
-		status, ok, warning, managed := ItemStatus(item)
-		tmp := make(map[string]string)
-		tmp["name"] = item.Name
-		tmp["status"] = status
-		tmp["local_version"] = item.LocalVersion
-		tmp["local_path"] = item.LocalPath
-		tmp["description"] = item.Description
-		if !managed {
-			tmp["utf8_status"] = fmt.Sprintf("%v  %s", emoji.House, status)
-		} else if !item.Installed {
-			tmp["utf8_status"] = fmt.Sprintf("%v  %s", emoji.Prohibited, status)
-		} else if warning {
-			tmp["utf8_status"] = fmt.Sprintf("%v  %s", emoji.Warning, status)
-		} else if ok {
-			tmp["utf8_status"] = fmt.Sprintf("%v  %s", emoji.CheckMark, status)
-		}
-		ret = append(ret, tmp)
+		ret = append(ret, item.toHubStatus())
 	}
 	return ret
 }

pkg/cwhub/cwhub_test.go

@@ -321,10 +321,10 @@ func TestInstallParser(t *testing.T) {
 	for _, it := range hubIdx[PARSERS] {
 		testInstallItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
-		_ = HubStatus(PARSERS, it.Name, false)
+		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
 		testTaintItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
-		_ = HubStatus(PARSERS, it.Name, false)
+		_ = GetHubStatusForItemType(PARSERS, it.Name, false)
 		testUpdateItem(cfg.Hub, t, it)
 		it = hubIdx[PARSERS][it.Name]
 		testDisableItem(cfg.Hub, t, it)
@@ -361,7 +361,7 @@ func TestInstallCollection(t *testing.T) {
 		testDisableItem(cfg.Hub, t, it)
 
 		it = hubIdx[COLLECTIONS][it.Name]
-		x := HubStatus(COLLECTIONS, it.Name, false)
+		x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
 		log.Printf("%+v", x)
 		break
 	}

scripts/func_tests/tests_post-install_2collections.sh

@@ -10,7 +10,7 @@ ${CSCLI_BIN} collections list || fail "failed to list collections"
 BASE_COLLECTION_COUNT=2
 
 # we expect 1 collections : linux 
-${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
+${CSCLI_BIN} collections list -ojson | ${JQ} ".collections | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
 
 # install an extra collection
 ${CSCLI} collections install crowdsecurity/mysql || fail "failed to install collection"
@@ -18,7 +18,7 @@ ${CSCLI} collections install crowdsecurity/mysql || fail "failed to install coll
 BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT+1))
 
 # we should now have 2 collections :)
-${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
+${CSCLI_BIN} collections list -ojson | ${JQ} ".collections | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
 
 # remove the collection
 ${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collection"
@@ -26,5 +26,5 @@ ${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collec
 BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT-1))
 
 # we expect 1 collections : linux 
-${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"
+${CSCLI_BIN} collections list -ojson | ${JQ} ".collections | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"