diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index 6806d39a7..158b80823 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -12,11 +12,15 @@ import ( ) func NewCollectionsCmd() *cobra.Command { - var cmdCollections = &cobra.Command{ - Use: "collections [action]", - Short: "Manage collections from hub", - Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`, - /*TBD fix help*/ + cmdCollections := &cobra.Command{ + Use: "collections [collection]...", + Short: "Manage hub collections", + Example: `cscli collections list -a +cscli collections install crowdsecurity/http-cve crowdsecurity/iptables +cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables +cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables +cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables +`, Args: cobra.MinimumNArgs(1), Aliases: []string{"collection"}, DisableAutoGenTag: true, @@ -35,142 +39,280 @@ func NewCollectionsCmd() *cobra.Command { }, } - var ignoreError bool + cmdCollections.AddCommand(NewCollectionsInstallCmd()) + cmdCollections.AddCommand(NewCollectionsRemoveCmd()) + cmdCollections.AddCommand(NewCollectionsUpgradeCmd()) + cmdCollections.AddCommand(NewCollectionsInspectCmd()) + cmdCollections.AddCommand(NewCollectionsListCmd()) - var cmdCollectionsInstall = &cobra.Command{ - Use: "install collection", - Short: "Install given collection(s)", - Long: `Fetch and install given collection(s) from hub`, - Example: `cscli collections install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), + return cmdCollections +} + +func runCollectionsInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.COLLECTIONS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) + Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewCollectionsInstallCmd() *cobra.Command { + cmdCollectionsInstall := &cobra.Command{ + Use: "install ...", + Short: "Install given collection(s)", + Long: `Fetch and install one or more collections from hub`, + Example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.COLLECTIONS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.COLLECTIONS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) - Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, + RunE: runCollectionsInstall, } - cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections") - cmdCollections.AddCommand(cmdCollectionsInstall) - var cmdCollectionsRemove = &cobra.Command{ - Use: "remove collection", + flags := cmdCollectionsInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple collections") + + return cmdCollectionsInstall +} + +func runCollectionsRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one collection to remove or '--all'") + } + + for _, name := range args { + if !force { + item := cwhub.GetItem(cwhub.COLLECTIONS, name) + if item == nil { + // XXX: this should be in GetItem? + return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS) + } + if len(item.BelongsToCollections) > 0 { + log.Warningf("%s belongs to other collections: %s", name, item.BelongsToCollections) + log.Warningf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection", name) + continue + } + } + + err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + +func NewCollectionsRemoveCmd() *cobra.Command { + cmdCollectionsRemove := &cobra.Command{ + Use: "remove ...", Short: "Remove given collection(s)", - Long: `Remove given collection(s) from hub`, - Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`, + Long: `Remove one or more collections`, + Example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to remove or '--all'") - } - - for _, name := range args { - if !forceAction { - item := cwhub.GetItem(cwhub.COLLECTIONS, name) - if item == nil { - return fmt.Errorf("unable to retrieve: %s", name) - } - if len(item.BelongsToCollections) > 0 { - log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections) - log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name) - continue - } - } - cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction) - } - return nil - }, + RunE: runCollectionsRemove, } - cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections") - cmdCollections.AddCommand(cmdCollectionsRemove) - var cmdCollectionsUpgrade = &cobra.Command{ - Use: "upgrade collection", + flags := cmdCollectionsRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the collections") + + return cmdCollectionsRemove +} + +func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one collection to upgrade or '--all'") + } + + for _, name := range args { + if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force); err != nil { + return err + } + } + + return nil +} + +func NewCollectionsUpgradeCmd() *cobra.Command { + cmdCollectionsUpgrade := &cobra.Command{ + Use: "upgrade ...", Short: "Upgrade given collection(s)", - Long: `Fetch and upgrade given collection(s) from hub`, - Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and upgrade one or more collections from the hub`, + Example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one collection to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction) - } - } - return nil - }, + RunE: runCollectionsUpgrade, } - cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections") - cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") - cmdCollections.AddCommand(cmdCollectionsUpgrade) - var cmdCollectionsInspect = &cobra.Command{ - Use: "inspect collection", - Short: "Inspect given collection", - Long: `Inspect given collection`, - Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`, + flags := cmdCollectionsUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the collections") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") + + return cmdCollectionsUpgrade +} + +func runCollectionsInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.COLLECTIONS, noMetrics); err != nil { + return err + } + } + + return nil +} + +func NewCollectionsInspectCmd() *cobra.Command { + cmdCollectionsInspect := &cobra.Command{ + Use: "inspect ...", + Short: "Inspect given collection(s)", + Long: `Inspect one or more collections`, + Example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.COLLECTIONS, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - for _, name := range args { - InspectItem(name, cwhub.COLLECTIONS) - } - }, + RunE: runCollectionsInspect, } - cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") - cmdCollections.AddCommand(cmdCollectionsInspect) - var cmdCollectionsList = &cobra.Command{ - Use: "list collection [-a]", - Short: "List all collections", - Long: `List all collections`, - Example: `cscli collections list`, - Args: cobra.ExactArgs(0), - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all) - }, - } - cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - cmdCollections.AddCommand(cmdCollectionsList) + flags := cmdCollectionsInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") - return cmdCollections + return cmdCollectionsInspect +} + +func runCollectionsList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if err = ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all); err != nil { + return err + } + + return nil +} + +func NewCollectionsListCmd() *cobra.Command { + cmdCollectionsList := &cobra.Command{ + Use: "list [collection... | -a]", + Short: "List collections", + Long: `List of installed/available/specified collections`, + Example: `cscli collections list +cscli collections list -a +cscli collections list crowdsecurity/http-cve crowdsecurity/iptables`, + DisableAutoGenTag: true, + RunE: runCollectionsList, + } + + flags := cmdCollectionsList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmdCollectionsList } diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 436eff8aa..73d1aca03 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -44,7 +44,7 @@ func backupHub(dirPath string) error { //for the local/tainted ones, we backup the full file if v.Tainted || v.Local || !v.UpToDate { //we need to backup stages for parsers - if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW { + if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS { fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil { return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 395e943bc..ccd9ebc5e 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -27,10 +27,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if item == nil { return "", fmt.Errorf("error retrieving item") } - if downloadOnly && item.Downloaded && item.UpToDate { - return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil - } - err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false) + err := cwhub.DownloadLatest(csConfig.Hub, item, false, false) if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } @@ -38,9 +35,6 @@ func silentInstallItem(name string, obtype string) (string, error) { return "", err } - if downloadOnly { - return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil - } err = cwhub.EnableItem(csConfig.Hub, item) if err != nil { return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) @@ -54,10 +48,6 @@ func silentInstallItem(name string, obtype string) (string, error) { func restoreHub(dirPath string) error { var err error - if err := csConfig.LoadHub(); err != nil { - return err - } - cwhub.SetHubBranch() for _, itype := range cwhub.ItemTypes { @@ -98,7 +88,7 @@ func restoreHub(dirPath string) error { if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { continue } - if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW { + if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS { //we expect a stage here if !file.IsDir() { continue diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 7bdfd5162..501fbc498 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -15,17 +15,14 @@ import ( func NewHubCmd() *cobra.Command { var cmdHub = &cobra.Command{ Use: "hub [action]", - Short: "Manage Hub", - Long: ` -Hub management + Short: "Manage hub index", + Long: `Hub management List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net). -The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. - `, - Example: ` -cscli hub list # List all installed configurations -cscli hub update # Download list of available configurations from the hub - `, +The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`, + Example: `cscli hub list +cscli hub update +cscli hub upgrade`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { @@ -45,39 +42,76 @@ cscli hub update # Download list of available configurations from the hub return cmdHub } +func runHubList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if err = require.Hub(csConfig); err != nil { + return err + } + + // use LocalSync to get warnings about tainted / outdated items + warn, _ := cwhub.LocalSync(csConfig.Hub) + for _, v := range warn { + log.Info(v) + } + + cwhub.DisplaySummary() + + err = ListItems(color.Output, []string{ + cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS, + }, nil, true, false, all) + if err != nil { + return err + } + + return nil +} + func NewHubListCmd() *cobra.Command { var cmdHubList = &cobra.Command{ Use: "list [-a]", - Short: "List installed configs", + Short: "List all installed configurations", Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) - for _, v := range warn { - log.Info(v) - } - cwhub.DisplaySummary() - ListItems(color.Output, []string{ - cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, - }, args, true, false, all) - - return nil - }, + RunE: runHubList, } - cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + + flags := cmdHubList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdHubList } +func runHubUpdate(cmd *cobra.Command, args []string) error { + if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + if !errors.Is(err, cwhub.ErrIndexNotFound) { + return fmt.Errorf("failed to get Hub index : %w", err) + } + log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) + cwhub.HubBranch = "master" + if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + return fmt.Errorf("failed to get Hub index after retry: %w", err) + } + } + + // use LocalSync to get warnings about tainted / outdated items + warn, _ := cwhub.LocalSync(csConfig.Hub) + for _, v := range warn { + log.Info(v) + } + + return nil +} + func NewHubUpdateCmd() *cobra.Command { var cmdHubUpdate = &cobra.Command{ Use: "update", - Short: "Fetch available configs from hub", + Short: "Download the latest index (catalog of available configurations)", Long: ` Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs. `, @@ -92,37 +126,51 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde return nil }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadHub(); err != nil { - return err - } - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { - if !errors.Is(err, cwhub.ErrIndexNotFound) { - return fmt.Errorf("failed to get Hub index : %w", err) - } - log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) - cwhub.HubBranch = "master" - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { - return fmt.Errorf("failed to get Hub index after retry: %w", err) - } - } - // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) - for _, v := range warn { - log.Info(v) - } - - return nil - }, + RunE: runHubUpdate, } return cmdHubUpdate } +func runHubUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + if err := require.Hub(csConfig); err != nil { + return err + } + + log.Infof("Upgrading collections") + if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + return err + } + + log.Infof("Upgrading parsers") + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + return err + } + + log.Infof("Upgrading scenarios") + if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + return err + } + + log.Infof("Upgrading postoverflows") + if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { + return err + } + + return nil +} + func NewHubUpgradeCmd() *cobra.Command { var cmdHubUpgrade = &cobra.Command{ Use: "upgrade", - Short: "Upgrade all configs installed from hub", + Short: "Upgrade all configurations to their latest version", Long: ` Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. `, @@ -137,24 +185,11 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if return nil }, - RunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { - return err - } - - log.Infof("Upgrading collections") - cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction) - log.Infof("Upgrading parsers") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction) - log.Infof("Upgrading scenarios") - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction) - log.Infof("Upgrading postoverflows") - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction) - - return nil - }, + RunE: runHubUpgrade, } - cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + + flags := cmdHubUpgrade.Flags() + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdHubUpgrade } diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go new file mode 100644 index 000000000..26caad655 --- /dev/null +++ b/cmd/crowdsec-cli/item_metrics.go @@ -0,0 +1,247 @@ +package main + +import ( + "fmt" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/fatih/color" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/prom2json" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/trace" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +func ShowMetrics(hubItem *cwhub.Item) { + switch hubItem.Type { + case cwhub.PARSERS: + metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) + parserMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.SCENARIOS: + metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) + scenarioMetricsTable(color.Output, hubItem.Name, metrics) + case cwhub.COLLECTIONS: + for _, item := range hubItem.Parsers { + metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, item) + parserMetricsTable(color.Output, item, metrics) + } + for _, item := range hubItem.Scenarios { + metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, item) + scenarioMetricsTable(color.Output, item, metrics) + } + for _, item := range hubItem.Collections { + hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) + if hubItem == nil { + log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) + } + ShowMetrics(hubItem) + } + case cwhub.WAAP_RULES: + log.Fatalf("FIXME: not implemented yet") + default: + log.Errorf("item of type '%s' is unknown", hubItem.Type) + } +} + +// GetParserMetric is a complete rip from prom2json +func GetParserMetric(url string, itemName string) map[string]map[string]int { + stats := make(map[string]map[string]int) + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + source, ok := metric.Labels["source"] + if !ok { + log.Debugf("no source in Metric %v", metric.Labels) + } else { + if srctype, ok := metric.Labels["type"]; ok { + source = srctype + ":" + source + } + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_reader_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + stats[source]["parsed"] = 0 + stats[source]["reads"] = 0 + stats[source]["unparsed"] = 0 + stats[source]["hits"] = 0 + } + stats[source]["reads"] += ival + case "cs_parser_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_parser_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + case "cs_node_hits_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["hits"] += ival + case "cs_node_hits_ok_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["parsed"] += ival + case "cs_node_hits_ko_total": + if _, ok := stats[source]; !ok { + stats[source] = make(map[string]int) + } + stats[source]["unparsed"] += ival + default: + continue + } + } + } + return stats +} + +func GetScenarioMetric(url string, itemName string) map[string]int { + stats := make(map[string]int) + + stats["instantiation"] = 0 + stats["curr_count"] = 0 + stats["overflow"] = 0 + stats["pour"] = 0 + stats["underflow"] = 0 + + result := GetPrometheusMetric(url) + for idx, fam := range result { + if !strings.HasPrefix(fam.Name, "cs_") { + continue + } + log.Tracef("round %d", idx) + for _, m := range fam.Metrics { + metric, ok := m.(prom2json.Metric) + if !ok { + log.Debugf("failed to convert metric to prom2json.Metric") + continue + } + name, ok := metric.Labels["name"] + if !ok { + log.Debugf("no name in Metric %v", metric.Labels) + } + if name != itemName { + continue + } + value := m.(prom2json.Metric).Value + fval, err := strconv.ParseFloat(value, 32) + if err != nil { + log.Errorf("Unexpected int value %s : %s", value, err) + continue + } + ival := int(fval) + + switch fam.Name { + case "cs_bucket_created_total": + stats["instantiation"] += ival + case "cs_buckets": + stats["curr_count"] += ival + case "cs_bucket_overflowed_total": + stats["overflow"] += ival + case "cs_bucket_poured_total": + stats["pour"] += ival + case "cs_bucket_underflowed_total": + stats["underflow"] += ival + default: + continue + } + } + } + return stats +} + +func GetPrometheusMetric(url string) []*prom2json.Family { + mfChan := make(chan *dto.MetricFamily, 1024) + + // Start with the DefaultTransport for sane defaults. + transport := http.DefaultTransport.(*http.Transport).Clone() + // Conservatively disable HTTP keep-alives as this program will only + // ever need a single HTTP request. + transport.DisableKeepAlives = true + // Timeout early if the server doesn't even return the headers. + transport.ResponseHeaderTimeout = time.Minute + + go func() { + defer trace.CatchPanic("crowdsec/GetPrometheusMetric") + err := prom2json.FetchMetricFamilies(url, mfChan, transport) + if err != nil { + log.Fatalf("failed to fetch prometheus metrics : %v", err) + } + }() + + result := []*prom2json.Family{} + for mf := range mfChan { + result = append(result, prom2json.NewFamily(mf)) + } + log.Debugf("Finished reading prometheus output, %d entries", len(result)) + + return result +} + +type unit struct { + value int64 + symbol string +} + +var ranges = []unit{ + {value: 1e18, symbol: "E"}, + {value: 1e15, symbol: "P"}, + {value: 1e12, symbol: "T"}, + {value: 1e9, symbol: "G"}, + {value: 1e6, symbol: "M"}, + {value: 1e3, symbol: "k"}, + {value: 1, symbol: ""}, +} + +func formatNumber(num int) string { + goodUnit := unit{} + for _, u := range ranges { + if int64(num) >= u.value { + goodUnit = u + break + } + } + + if goodUnit.value == 1 { + return fmt.Sprintf("%d%s", num, goodUnit.symbol) + } + + res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 + return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) +} diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go new file mode 100644 index 000000000..c52239fb4 --- /dev/null +++ b/cmd/crowdsec-cli/item_suggest.go @@ -0,0 +1,93 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/agext/levenshtein" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "slices" + + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +const MaxDistance = 7 + +func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { + errMsg := "" + if score < MaxDistance { + errMsg = fmt.Sprintf("can't find '%s' in %s, did you mean %s?", baseItem, itemType, suggestItem) + } else { + errMsg = fmt.Sprintf("can't find '%s' in %s", baseItem, itemType) + } + if ignoreErr { + log.Error(errMsg) + } else { + log.Fatalf(errMsg) + } +} + +func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { + allItems := make([]string, 0) + nearestScore := 100 + nearestItem := &cwhub.Item{} + hubItems := cwhub.GetItemMap(itemType) + for _, item := range hubItems { + allItems = append(allItems, item.Name) + } + + for _, s := range allItems { + d := levenshtein.Distance(itemName, s, nil) + if d < nearestScore { + nearestScore = d + nearestItem = cwhub.GetItem(itemType, s) + } + } + return nearestItem, nearestScore +} + +func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if err := require.Hub(csConfig); err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + hubItems := cwhub.GetItemMap(itemType) + for _, item := range hubItems { + if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { + comp = append(comp, item.Name) + } + } + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + return comp, cobra.ShellCompDirectiveNoFileComp +} + +func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if err := require.Hub(csConfig); err != nil { + return nil, cobra.ShellCompDirectiveDefault + } + + items, err := cwhub.GetInstalledItemsAsString(itemType) + if err != nil { + cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) + return nil, cobra.ShellCompDirectiveDefault + } + + comp := make([]string, 0) + + if toComplete != "" { + for _, item := range items { + if strings.Contains(item, toComplete) { + comp = append(comp, item) + } + } + } else { + comp = items + } + + cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) + + return comp, cobra.ShellCompDirectiveNoFileComp +} diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go new file mode 100644 index 000000000..cebf5b723 --- /dev/null +++ b/cmd/crowdsec-cli/items.go @@ -0,0 +1,174 @@ +package main + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "io" + "slices" + "sort" + "strings" + + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + + +func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) { + itemNames := cwhub.GetItemNames(itemType) + + notExist := []string{} + if len(args) > 0 { + installedOnly = false + for _, arg := range args { + if !slices.Contains(itemNames, arg) { + notExist = append(notExist, arg) + } + } + } + + if len(notExist) > 0 { + return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType) + } + + if len(args) > 0 { + itemNames = args + } + + if installedOnly { + installed := []string{} + for _, item := range itemNames { + if cwhub.GetItem(itemType, item).Installed { + installed = append(installed, item) + } + } + return installed, nil + } + return itemNames, nil +} + + +func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { + var err error + items := make(map[string][]string) + for _, itemType := range itemTypes { + if items[itemType], err = selectItems(itemType, args, !all); err != nil { + return err + } + } + + if csConfig.Cscli.Output == "human" { + for _, itemType := range itemTypes { + listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType]) + } + } else if csConfig.Cscli.Output == "json" { + type itemHubStatus struct { + Name string `json:"name"` + LocalVersion string `json:"local_version"` + LocalPath string `json:"local_path"` + Description string `json:"description"` + UTF8Status string `json:"utf8_status"` + Status string `json:"status"` + } + + hubStatus := make(map[string][]itemHubStatus) + for _, itemType := range itemTypes { + // empty slice in case there are no items of this type + hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) + for i, itemName := range items[itemType] { + item := cwhub.GetItem(itemType, itemName) + status, emo := item.Status() + hubStatus[itemType][i] = itemHubStatus{ + Name: item.Name, + LocalVersion: item.LocalVersion, + LocalPath: item.LocalPath, + Description: item.Description, + Status: status, + UTF8Status: fmt.Sprintf("%v %s", emo, status), + } + } + h := hubStatus[itemType] + sort.Slice(h, func(i, j int) bool { return h[i].Name < h[j].Name }) + } + x, err := json.MarshalIndent(hubStatus, "", " ") + if err != nil { + log.Fatalf("failed to unmarshal") + } + out.Write(x) + } else if csConfig.Cscli.Output == "raw" { + csvwriter := csv.NewWriter(out) + if showHeader { + header := []string{"name", "status", "version", "description"} + if showType { + header = append(header, "type") + } + err := csvwriter.Write(header) + if err != nil { + log.Fatalf("failed to write header: %s", err) + } + + } + for _, itemType := range itemTypes { + for _, itemName := range items[itemType] { + item := cwhub.GetItem(itemType, itemName) + status, _ := item.Status() + if item.LocalVersion == "" { + item.LocalVersion = "n/a" + } + row := []string{ + item.Name, + status, + item.LocalVersion, + item.Description, + } + if showType { + row = append(row, itemType) + } + err := csvwriter.Write(row) + if err != nil { + log.Fatalf("failed to write raw output : %s", err) + } + } + } + csvwriter.Flush() + } + return nil +} + +func InspectItem(name string, itemType string, noMetrics bool) error { + hubItem := cwhub.GetItem(itemType, name) + if hubItem == nil { + return fmt.Errorf("can't find '%s' in %s", name, itemType) + } + + var ( + b []byte + err error + ) + + switch csConfig.Cscli.Output { + case "human", "raw": + b, err = yaml.Marshal(*hubItem) + if err != nil { + return fmt.Errorf("unable to marshal item: %s", err) + } + case "json": + b, err = json.MarshalIndent(*hubItem, "", " ") + if err != nil { + return fmt.Errorf("unable to marshal item: %s", err) + } + } + + fmt.Printf("%s", string(b)) + + if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { + return nil + } + + fmt.Printf("\nCurrent metrics: \n") + ShowMetrics(hubItem) + + return nil +} diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index b7c5da886..59f737d30 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -29,13 +29,6 @@ var dbClient *database.Client var OutputFormat string var OutputColor string -var downloadOnly bool -var forceAction bool -var purge bool -var all bool - -var prometheusURL string - var mergedConfig string func initConfig() { @@ -58,10 +51,8 @@ func initConfig() { if err != nil { log.Fatal(err) } - if err := csConfig.LoadCSCLI(); err != nil { - log.Fatal(err) - } } else { + // XXX: check all the defaults csConfig = csconfig.NewDefaultConfig() } @@ -255,7 +246,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) - rootCmd.AddCommand(NewWafRulesCmd()) + rootCmd.AddCommand(NewWaapRulesCmd()) if fflag.CscliSetup.IsEnabled() { rootCmd.AddCommand(NewSetupCmd()) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 8ab3f01bd..a03614aae 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -284,8 +284,20 @@ var noUnit bool func runMetrics(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadPrometheus(); err != nil { - return fmt.Errorf("failed to load prometheus config: %w", err) + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noUnit, err = flags.GetBool("no-unit") + if err != nil { + return err } if csConfig.Prometheus == nil { @@ -296,17 +308,8 @@ func runMetrics(cmd *cobra.Command, args []string) error { return fmt.Errorf("prometheus is not enabled, can't show metrics") } - if prometheusURL == "" { - prometheusURL = csConfig.Cscli.PrometheusUrl - } - - if prometheusURL == "" { - return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath) - } - - err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output) - if err != nil { - return fmt.Errorf("could not fetch prometheus metrics: %w", err) + if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil { + return err } return nil } @@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command { DisableAutoGenTag: true, RunE: runMetrics, } - cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://:/metrics)") - cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units") + + flags := cmdMetrics.PersistentFlags() + flags.StringP("url", "u", "", "Prometheus url (http://:/metrics)") + flags.Bool("no-unit", false, "Show the real number instead of formatted with units") return cmdMetrics } diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index d97b070db..49e301a6c 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -12,14 +12,14 @@ import ( ) func NewParsersCmd() *cobra.Command { - var cmdParsers = &cobra.Command{ - Use: "parsers [action] [config]", - Short: "Install/Remove/Upgrade/Inspect parser(s) from hub", - Example: `cscli parsers install crowdsecurity/sshd-logs -cscli parsers inspect crowdsecurity/sshd-logs -cscli parsers upgrade crowdsecurity/sshd-logs -cscli parsers list -cscli parsers remove crowdsecurity/sshd-logs + cmdParsers := &cobra.Command{ + Use: "parsers [parser]...", + Short: "Manage hub parsers", + Example: `cscli parsers list -a +cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs +cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs `, Args: cobra.MinimumNArgs(1), Aliases: []string{"parser"}, @@ -48,147 +48,258 @@ cscli parsers remove crowdsecurity/sshd-logs return cmdParsers } -func NewParsersInstallCmd() *cobra.Command { - var ignoreError bool +func runParsersInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - var cmdParsersInstall = &cobra.Command{ - Use: "install [config]", + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.PARSERS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.PARSERS, name) + Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewParsersInstallCmd() *cobra.Command { + cmdParsersInstall := &cobra.Command{ + Use: "install ...", Short: "Install given parser(s)", - Long: `Fetch and install given parser(s) from hub`, - Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and install one or more parsers from the hub`, + Example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.PARSERS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS, name) - Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, + RunE: runParsersInstall, } - cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers") + flags := cmdParsersInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple parsers") return cmdParsersInstall } +func runParsersRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one parser to remove or '--all'") + } + + for _, name := range args { + err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + func NewParsersRemoveCmd() *cobra.Command { cmdParsersRemove := &cobra.Command{ - Use: "remove [config]", + Use: "remove ...", Short: "Remove given parser(s)", - Long: `Remove given parse(s) from hub`, - Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`, + Long: `Remove one or more parsers`, + Example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to remove or '--all'") - } - - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction) - } - - return nil - }, + RunE: runParsersRemove, } - cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers") + flags := cmdParsersRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the parsers") return cmdParsersRemove } +func runParsersUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one parser to upgrade or '--all'") + } + + for _, name := range args { + if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force); err != nil { + return err + } + } + + return nil +} + func NewParsersUpgradeCmd() *cobra.Command { cmdParsersUpgrade := &cobra.Command{ - Use: "upgrade [config]", + Use: "upgrade ...", Short: "Upgrade given parser(s)", - Long: `Fetch and upgrade given parser(s) from hub`, - Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and upgrade one or more parsers from the hub`, + Example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS, args, toComplete) }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one parser to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction) - } - } - return nil - }, + RunE: runParsersUpgrade, } - cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers") - cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + flags := cmdParsersUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the parsers") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdParsersUpgrade } +func runParsersInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.PARSERS, noMetrics); err != nil { + return err + } + } + + return nil +} + func NewParsersInspectCmd() *cobra.Command { - var cmdParsersInspect = &cobra.Command{ - Use: "inspect [name]", - Short: "Inspect given parser", - Long: `Inspect given parser`, - Example: `cscli parsers inspect crowdsec/xxx`, - DisableAutoGenTag: true, + cmdParsersInspect := &cobra.Command{ + Use: "inspect ", + Short: "Inspect a parser", + Long: `Inspect a parser`, + Example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`, Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.PARSERS, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.PARSERS) - }, + RunE: runParsersInspect, } - cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") + flags := cmdParsersInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") return cmdParsersInspect } -func NewParsersListCmd() *cobra.Command { - var cmdParsersList = &cobra.Command{ - Use: "list [name]", - Short: "List all parsers or given one", - Long: `List all parsers or given one`, - Example: `cscli parsers list -cscli parser list crowdsecurity/xxx`, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all) - }, +func runParsersList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err } - cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + if err = ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all); err != nil { + return err + } + + return nil +} + +func NewParsersListCmd() *cobra.Command { + cmdParsersList := &cobra.Command{ + Use: "list [parser... | -a]", + Short: "List parsers", + Long: `List of installed/available/specified parsers`, + Example: `cscli parsers list +cscli parsers list -a +cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs`, + DisableAutoGenTag: true, + RunE: runParsersList, + } + + flags := cmdParsersList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdParsersList } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index f4db0a79e..8c5f1b175 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -13,13 +13,14 @@ import ( func NewPostOverflowsCmd() *cobra.Command { cmdPostOverflows := &cobra.Command{ - Use: "postoverflows [action] [config]", - Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub", - Example: `cscli postoverflows install crowdsecurity/cdn-whitelist - cscli postoverflows inspect crowdsecurity/cdn-whitelist - cscli postoverflows upgrade crowdsecurity/cdn-whitelist - cscli postoverflows list - cscli postoverflows remove crowdsecurity/cdn-whitelist`, + Use: "postoverflows [postoverflow]...", + Short: "Manage hub postoverflows", + Example: `cscli postoverflows list -a +cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns +cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns +`, Args: cobra.MinimumNArgs(1), Aliases: []string{"postoverflow"}, DisableAutoGenTag: true, @@ -47,145 +48,259 @@ func NewPostOverflowsCmd() *cobra.Command { return cmdPostOverflows } -func NewPostOverflowsInstallCmd() *cobra.Command { - var ignoreError bool +func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name) + Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewPostOverflowsInstallCmd() *cobra.Command { cmdPostOverflowsInstall := &cobra.Command{ - Use: "install [config]", + Use: "install ...", Short: "Install given postoverflow(s)", - Long: `Fetch and install given postoverflow(s) from hub`, - Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`, + Long: `Fetch and install one or more postoverflows from the hub`, + Example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name) - Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil + return compAllItems(cwhub.POSTOVERFLOWS, args, toComplete) }, + RunE: runPostOverflowsInstall, } - cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows") + flags := cmdPostOverflowsInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple postoverflows") return cmdPostOverflowsInstall } +func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one postoverflow to remove or '--all'") + } + + for _, name := range args { + err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + func NewPostOverflowsRemoveCmd() *cobra.Command { cmdPostOverflowsRemove := &cobra.Command{ - Use: "remove [config]", + Use: "remove ...", Short: "Remove given postoverflow(s)", - Long: `remove given postoverflow(s)`, - Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`, + Long: `remove one or more postoverflows from the hub`, + Example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`, Aliases: []string{"delete"}, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to remove or '--all'") - } - - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction) - } - - return nil + return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) }, + RunE: runPostOverflowsRemove, } - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows") + flags := cmdPostOverflowsRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Delete all the postoverflows") return cmdPostOverflowsRemove } -func NewPostOverflowsUpgradeCmd() *cobra.Command { - cmdPostOverflowsUpgrade := &cobra.Command{ - Use: "upgrade [config]", - Short: "Upgrade given postoverflow(s)", - Long: `Fetch and Upgrade given postoverflow(s) from hub`, - Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`, - DisableAutoGenTag: true, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction) - } - } - return nil - }, +func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err } - cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows") - cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'") + } + + for _, name := range args { + if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil { + return err + } + } + + return nil +} + +func NewPostOverflowsUpgradeCmd() *cobra.Command { + cmdPostOverflowsUpgrade := &cobra.Command{ + Use: "upgrade ...", + Short: "Upgrade given postoverflow(s)", + Long: `Fetch and upgrade one or more postoverflows from the hub`, + Example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + DisableAutoGenTag: true, + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) + }, + RunE: runPostOverflowUpgrade, + } + + flags := cmdPostOverflowsUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the postoverflows") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdPostOverflowsUpgrade } +func runPostOverflowsInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.POSTOVERFLOWS, noMetrics); err != nil { + return err + } + } + + return nil +} + func NewPostOverflowsInspectCmd() *cobra.Command { cmdPostOverflowsInspect := &cobra.Command{ - Use: "inspect [config]", - Short: "Inspect given postoverflow", - Long: `Inspect given postoverflow`, - Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`, - DisableAutoGenTag: true, + Use: "inspect ", + Short: "Inspect a postoverflow", + Long: `Inspect a postoverflow`, + Example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`, Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete) - }, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.PARSERS_OVFLW) + return compInstalledItems(cwhub.POSTOVERFLOWS, args, toComplete) }, + RunE: runPostOverflowsInspect, } + flags := cmdPostOverflowsInspect.Flags() + + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") + return cmdPostOverflowsInspect } -func NewPostOverflowsListCmd() *cobra.Command { - cmdPostOverflowsList := &cobra.Command{ - Use: "list [config]", - Short: "List all postoverflows or given one", - Long: `List all postoverflows or given one`, - Example: `cscli postoverflows list -cscli postoverflows list crowdsecurity/xxx`, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all) - }, +func runPostOverflowsList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err } - cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + if err = ListItems(color.Output, []string{cwhub.POSTOVERFLOWS}, args, false, true, all); err != nil { + return err + } + + return nil +} + +func NewPostOverflowsListCmd() *cobra.Command { + cmdPostOverflowsList := &cobra.Command{ + Use: "list [postoverflow]...", + Short: "List postoverflows", + Long: `List of installed/available/specified postoverflows`, + Example: `cscli postoverflows list +cscli postoverflows list -a +cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns`, + DisableAutoGenTag: true, + RunE: runPostOverflowsList, + } + + flags := cmdPostOverflowsList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdPostOverflowsList } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index f4129a44f..fba59cb1e 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -65,10 +65,6 @@ func Notifications(c *csconfig.Config) error { } func Hub (c *csconfig.Config) error { - if err := c.LoadHub(); err != nil { - return err - } - if c.Hub == nil { return fmt.Errorf("you must configure cli before interacting with hub") } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 01e0b02dc..863e50236 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -12,14 +12,14 @@ import ( ) func NewScenariosCmd() *cobra.Command { - var cmdScenarios = &cobra.Command{ - Use: "scenarios [action] [config]", - Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub", - Example: `cscli scenarios list [-a] -cscli scenarios install crowdsecurity/ssh-bf -cscli scenarios inspect crowdsecurity/ssh-bf -cscli scenarios upgrade crowdsecurity/ssh-bf -cscli scenarios remove crowdsecurity/ssh-bf + cmdScenarios := &cobra.Command{ + Use: "scenarios [scenario]...", + Short: "Manage hub scenarios", + Example: `cscli scenarios list -a +cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing +cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing `, Args: cobra.MinimumNArgs(1), Aliases: []string{"scenario"}, @@ -48,141 +48,258 @@ cscli scenarios remove crowdsecurity/ssh-bf return cmdScenarios } -func NewCmdScenariosInstall() *cobra.Command { - var ignoreError bool +func runScenariosInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - var cmdScenariosInstall = &cobra.Command{ - Use: "install [config]", - Short: "Install given scenario(s)", - Long: `Fetch and install given scenario(s) from hub`, - Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`, - Args: cobra.MinimumNArgs(1), + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.SCENARIOS, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.SCENARIOS, name) + Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewCmdScenariosInstall() *cobra.Command { + cmdScenariosInstall := &cobra.Command{ + Use: "install ...", + Short: "Install given scenario(s)", + Long: `Fetch and install one or more scenarios from the hub`, + Example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - for _, name := range args { - t := cwhub.GetItem(cwhub.SCENARIOS, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.SCENARIOS, name) - Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { - if !ignoreError { - return fmt.Errorf("error while installing '%s': %w", name, err) - } - log.Errorf("Error while installing '%s': %s", name, err) - } - } - return nil - }, + RunE: runScenariosInstall, } - cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios") + + flags := cmdScenariosInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple scenarios") return cmdScenariosInstall } +func runScenariosRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one scenario to remove or '--all'") + } + + for _, name := range args { + err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + func NewCmdScenariosRemove() *cobra.Command { - var cmdScenariosRemove = &cobra.Command{ - Use: "remove [config]", - Short: "Remove given scenario(s)", - Long: `remove given scenario(s)`, - Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`, - Aliases: []string{"delete"}, + cmdScenariosRemove := &cobra.Command{ + Use: "remove ...", + Short: "Remove given scenario(s)", + Long: `remove one or more scenarios`, + Example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`, + Aliases: []string{"delete"}, + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction) - return nil - } - - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to remove or '--all'") - } - - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction) - } - return nil - }, + RunE: runScenariosRemove, } - cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios") + + flags := cmdScenariosRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the scenarios") return cmdScenariosRemove } +func runScenariosUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one scenario to upgrade or '--all'") + } + + for _, name := range args { + if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force); err != nil { + return err + } + } + + return nil +} + func NewCmdScenariosUpgrade() *cobra.Command { - var cmdScenariosUpgrade = &cobra.Command{ - Use: "upgrade [config]", - Short: "Upgrade given scenario(s)", - Long: `Fetch and Upgrade given scenario(s) from hub`, - Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`, + cmdScenariosUpgrade := &cobra.Command{ + Use: "upgrade ...", + Short: "Upgrade given scenario(s)", + Long: `Fetch and upgrade one or more scenarios from the hub`, + Example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`, + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction) - } else { - if len(args) == 0 { - return fmt.Errorf("specify at least one scenario to upgrade or '--all'") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction) - } - } - return nil - }, + RunE: runScenariosUpgrade, } - cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios") - cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") + + flags := cmdScenariosUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the scenarios") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") return cmdScenariosUpgrade } +func runScenariosInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.SCENARIOS, noMetrics); err != nil { + return err + } + } + + return nil +} + func NewCmdScenariosInspect() *cobra.Command { - var cmdScenariosInspect = &cobra.Command{ - Use: "inspect [config]", - Short: "Inspect given scenario", - Long: `Inspect given scenario`, - Example: `cscli scenarios inspect crowdsec/xxx`, - Args: cobra.MinimumNArgs(1), + cmdScenariosInspect := &cobra.Command{ + Use: "inspect ", + Short: "Inspect a scenario", + Long: `Inspect a scenario`, + Example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`, + Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.SCENARIOS, args, toComplete) }, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.SCENARIOS) - }, + RunE: runScenariosInspect, } - cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") + + flags := cmdScenariosInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") return cmdScenariosInspect } -func NewCmdScenariosList() *cobra.Command { - var cmdScenariosList = &cobra.Command{ - Use: "list [config]", - Short: "List all scenario(s) or given one", - Long: `List all scenario(s) or given one`, - Example: `cscli scenarios list -cscli scenarios list crowdsecurity/xxx`, - DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all) - }, +func runScenariosList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err } - cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") + + if err = ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all); err != nil { + return err + } + + return nil +} + +func NewCmdScenariosList() *cobra.Command { + cmdScenariosList := &cobra.Command{ + Use: "list [scenario]...", + Short: "List scenarios", + Long: `List of installed/available/specified scenarios`, + Example: `cscli scenarios list +cscli scenarios list -a +cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing`, + DisableAutoGenTag: true, + RunE: runScenariosList, + } + + flags := cmdScenariosList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") return cmdScenariosList } diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index e5a4c36ab..0c87ad3c9 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -58,10 +58,6 @@ func stripAnsiString(str string) string { func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") - err := csConfig.LoadPrometheus() - if err != nil { - return nil, nil, err - } if csConfig.Cscli.PrometheusUrl == "" { log.Warn("No Prometheus URL configured, metrics will not be collected") @@ -69,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) { } humanMetrics := bytes.NewBuffer(nil) - err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human") + err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human") if err != nil { return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err) } - req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil) + req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil) if err != nil { return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err) } @@ -135,7 +131,9 @@ func collectOSInfo() ([]byte, error) { func collectHubItems(itemType string) []byte { out := bytes.NewBuffer(nil) log.Infof("Collecting %s list", itemType) - ListItems(out, []string{itemType}, []string{}, false, true, all) + if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil { + log.Warnf("could not collect %s list: %s", itemType, err) + } return out.Bytes() } @@ -335,7 +333,7 @@ cscli support dump -f /tmp/crowdsec-support.zip if !skipHub { infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS) infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS) - infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW) + infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.POSTOVERFLOWS) infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS) } diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 1101235b6..eb7fb51e0 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -1,36 +1,17 @@ package main import ( - "encoding/csv" - "encoding/json" "fmt" - "io" - "math" "net" - "net/http" - "slices" - "strconv" "strings" - "time" - "github.com/agext/levenshtein" - "github.com/fatih/color" - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/prom2json" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "gopkg.in/yaml.v2" - "github.com/crowdsecurity/go-cs-lib/trace" - - "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/types" ) -const MaxDistance = 7 - func printHelp(cmd *cobra.Command) { err := cmd.Help() if err != nil { @@ -38,197 +19,6 @@ func printHelp(cmd *cobra.Command) { } } -func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) { - errMsg := "" - if score < MaxDistance { - errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem) - } else { - errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem) - } - if ignoreErr { - log.Error(errMsg) - } else { - log.Fatalf(errMsg) - } -} - -func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { - allItems := make([]string, 0) - nearestScore := 100 - nearestItem := &cwhub.Item{} - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - allItems = append(allItems, item.Name) - } - - for _, s := range allItems { - d := levenshtein.Distance(itemName, s, nil) - if d < nearestScore { - nearestScore = d - nearestItem = cwhub.GetItem(itemType, s) - } - } - return nearestItem, nearestScore -} - -func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - hubItems := cwhub.GetHubStatusForItemType(itemType, "", true) - for _, item := range hubItems { - if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { - comp = append(comp, item.Name) - } - } - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { - return nil, cobra.ShellCompDirectiveDefault - } - - items, err := cwhub.GetInstalledItemsAsString(itemType) - if err != nil { - cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) - return nil, cobra.ShellCompDirectiveDefault - } - - comp := make([]string, 0) - - if toComplete != "" { - for _, item := range items { - if strings.Contains(item, toComplete) { - comp = append(comp, item) - } - } - } else { - comp = items - } - - cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true) - - return comp, cobra.ShellCompDirectiveNoFileComp -} - -func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) { - var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus) - - for _, itemType := range itemTypes { - itemName := "" - if len(args) == 1 { - itemName = args[0] - } - hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all) - } - - if csConfig.Cscli.Output == "human" { - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses) - } - } else if csConfig.Cscli.Output == "json" { - x, err := json.MarshalIndent(hubStatusByItemType, "", " ") - if err != nil { - log.Fatalf("failed to unmarshal") - } - out.Write(x) - } else if csConfig.Cscli.Output == "raw" { - csvwriter := csv.NewWriter(out) - if showHeader { - header := []string{"name", "status", "version", "description"} - if showType { - header = append(header, "type") - } - err := csvwriter.Write(header) - if err != nil { - log.Fatalf("failed to write header: %s", err) - } - - } - for _, itemType := range itemTypes { - var statuses []cwhub.ItemHubStatus - var ok bool - if statuses, ok = hubStatusByItemType[itemType]; !ok { - log.Errorf("unknown item type: %s", itemType) - continue - } - for _, status := range statuses { - if status.LocalVersion == "" { - status.LocalVersion = "n/a" - } - row := []string{ - status.Name, - status.Status, - status.LocalVersion, - status.Description, - } - if showType { - row = append(row, itemType) - } - err := csvwriter.Write(row) - if err != nil { - log.Fatalf("failed to write raw output : %s", err) - } - } - } - csvwriter.Flush() - } -} - -func InspectItem(name string, objecitemType string) { - - hubItem := cwhub.GetItem(objecitemType, name) - if hubItem == nil { - log.Fatalf("unable to retrieve item.") - } - var b []byte - var err error - switch csConfig.Cscli.Output { - case "human", "raw": - b, err = yaml.Marshal(*hubItem) - if err != nil { - log.Fatalf("unable to marshal item : %s", err) - } - case "json": - b, err = json.MarshalIndent(*hubItem, "", " ") - if err != nil { - log.Fatalf("unable to marshal item : %s", err) - } - } - fmt.Printf("%s", string(b)) - if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" { - return - } - - if prometheusURL == "" { - //This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus - //But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case - listenAddr := csConfig.Prometheus.ListenAddr - if listenAddr == "" { - listenAddr = "127.0.0.1" - } - listenPort := csConfig.Prometheus.ListenPort - if listenPort == 0 { - listenPort = 6060 - } - prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort) - log.Debugf("No prometheus URL provided using: %s", prometheusURL) - } - - fmt.Printf("\nCurrent metrics : \n") - ShowMetrics(hubItem) -} - func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error { /*if a range is provided, change the scope*/ @@ -259,234 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value * return nil } -func ShowMetrics(hubItem *cwhub.Item) { - switch hubItem.Type { - case cwhub.PARSERS: - metrics := GetParserMetric(prometheusURL, hubItem.Name) - parserMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.SCENARIOS: - metrics := GetScenarioMetric(prometheusURL, hubItem.Name) - scenarioMetricsTable(color.Output, hubItem.Name, metrics) - case cwhub.COLLECTIONS: - for _, item := range hubItem.Parsers { - metrics := GetParserMetric(prometheusURL, item) - parserMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Scenarios { - metrics := GetScenarioMetric(prometheusURL, item) - scenarioMetricsTable(color.Output, item, metrics) - } - for _, item := range hubItem.Collections { - hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) - if hubItem == nil { - log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) - } - ShowMetrics(hubItem) - } - case cwhub.WAAP_RULES: - log.Fatalf("FIXME: not implemented yet") - default: - log.Errorf("item of type '%s' is unknown", hubItem.Type) - } -} - -// GetParserMetric is a complete rip from prom2json -func GetParserMetric(url string, itemName string) map[string]map[string]int { - stats := make(map[string]map[string]int) - - result := GetPrometheusMetric(url) - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - source, ok := metric.Labels["source"] - if !ok { - log.Debugf("no source in Metric %v", metric.Labels) - } else { - if srctype, ok := metric.Labels["type"]; ok { - source = srctype + ":" + source - } - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_reader_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - stats[source]["parsed"] = 0 - stats[source]["reads"] = 0 - stats[source]["unparsed"] = 0 - stats[source]["hits"] = 0 - } - stats[source]["reads"] += ival - case "cs_parser_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_parser_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - case "cs_node_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["hits"] += ival - case "cs_node_hits_ok_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["parsed"] += ival - case "cs_node_hits_ko_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - } - stats[source]["unparsed"] += ival - default: - continue - } - } - } - return stats -} - -func GetScenarioMetric(url string, itemName string) map[string]int { - stats := make(map[string]int) - - stats["instantiation"] = 0 - stats["curr_count"] = 0 - stats["overflow"] = 0 - stats["pour"] = 0 - stats["underflow"] = 0 - - result := GetPrometheusMetric(url) - for idx, fam := range result { - if !strings.HasPrefix(fam.Name, "cs_") { - continue - } - log.Tracef("round %d", idx) - for _, m := range fam.Metrics { - metric, ok := m.(prom2json.Metric) - if !ok { - log.Debugf("failed to convert metric to prom2json.Metric") - continue - } - name, ok := metric.Labels["name"] - if !ok { - log.Debugf("no name in Metric %v", metric.Labels) - } - if name != itemName { - continue - } - value := m.(prom2json.Metric).Value - fval, err := strconv.ParseFloat(value, 32) - if err != nil { - log.Errorf("Unexpected int value %s : %s", value, err) - continue - } - ival := int(fval) - - switch fam.Name { - case "cs_bucket_created_total": - stats["instantiation"] += ival - case "cs_buckets": - stats["curr_count"] += ival - case "cs_bucket_overflowed_total": - stats["overflow"] += ival - case "cs_bucket_poured_total": - stats["pour"] += ival - case "cs_bucket_underflowed_total": - stats["underflow"] += ival - default: - continue - } - } - } - return stats -} - -func GetPrometheusMetric(url string) []*prom2json.Family { - mfChan := make(chan *dto.MetricFamily, 1024) - - // Start with the DefaultTransport for sane defaults. - transport := http.DefaultTransport.(*http.Transport).Clone() - // Conservatively disable HTTP keep-alives as this program will only - // ever need a single HTTP request. - transport.DisableKeepAlives = true - // Timeout early if the server doesn't even return the headers. - transport.ResponseHeaderTimeout = time.Minute - - go func() { - defer trace.CatchPanic("crowdsec/GetPrometheusMetric") - err := prom2json.FetchMetricFamilies(url, mfChan, transport) - if err != nil { - log.Fatalf("failed to fetch prometheus metrics : %v", err) - } - }() - - result := []*prom2json.Family{} - for mf := range mfChan { - result = append(result, prom2json.NewFamily(mf)) - } - log.Debugf("Finished reading prometheus output, %d entries", len(result)) - - return result -} - -type unit struct { - value int64 - symbol string -} - -var ranges = []unit{ - {value: 1e18, symbol: "E"}, - {value: 1e15, symbol: "P"}, - {value: 1e12, symbol: "T"}, - {value: 1e9, symbol: "G"}, - {value: 1e6, symbol: "M"}, - {value: 1e3, symbol: "k"}, - {value: 1, symbol: ""}, -} - -func formatNumber(num int) string { - goodUnit := unit{} - for _, u := range ranges { - if int64(num) >= u.value { - goodUnit = u - break - } - } - - if goodUnit.value == 1 { - return fmt.Sprintf("%d%s", num, goodUnit.symbol) - } - - res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100 - return fmt.Sprintf("%.2f%s", res, goodUnit.symbol) -} - func getDBClient() (*database.Client, error) { var err error if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { @@ -520,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string { } return slice - } diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 16f42d72a..840250f9c 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -10,14 +10,16 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) { +func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) { t := newLightTable(out) t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) - for _, status := range statuses { - t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath) + for itemName := range itemNames { + item := cwhub.GetItem(itemType, itemNames[itemName]) + status, emo := item.Status() + t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) } renderTableTitle(out, title) t.Render() diff --git a/cmd/crowdsec-cli/waap_rules.go b/cmd/crowdsec-cli/waap_rules.go index 548921836..c2f3ade36 100644 --- a/cmd/crowdsec-cli/waap_rules.go +++ b/cmd/crowdsec-cli/waap_rules.go @@ -7,36 +7,28 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func NewWafRulesCmd() *cobra.Command { - var cmdWafRules = &cobra.Command{ - Use: "waap-rules [action] [config]", - Short: "Install/Remove/Upgrade/Inspect waf-rule(s) from hub", - Example: `cscli waap-rules install crowdsecurity/core-rule-set -cscli waap-rules inspect crowdsecurity/core-rule-set -cscli waap-rules upgrade crowdsecurity/core-rule-set -cscli waap-rules list -cscli waap-rules remove crowdsecurity/core-rule-set +func NewWaapRulesCmd() *cobra.Command { + cmdWaapRules := &cobra.Command{ + Use: "waap-rules [waap-rule]...", + Short: "Manage hub waap rules", + Example: `cscli waap-rules list -a +cscli waap-rules install crowdsecurity/crs +cscli waap-rules inspect crowdsecurity/crs +cscli waap-rules upgrade crowdsecurity/crs +cscli waap-rules remove crowdsecurity/crs `, Args: cobra.MinimumNArgs(1), Aliases: []string{"waap-rule"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadHub(); err != nil { - log.Fatal(err) - } - if csConfig.Hub == nil { - return fmt.Errorf("you must configure cli before interacting with hub") + if err := require.Hub(csConfig); err != nil { + return err } - cwhub.SetHubBranch() - - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - log.Info("Run 'sudo cscli hub update' to get the hub index") - log.Fatalf("Failed to get Hub index : %v", err) - } return nil }, PersistentPostRun: func(cmd *cobra.Command, args []string) { @@ -47,148 +39,267 @@ cscli waap-rules remove crowdsecurity/core-rule-set }, } - cmdWafRules.AddCommand(NewWafRulesInstallCmd()) - cmdWafRules.AddCommand(NewWafRulesRemoveCmd()) - cmdWafRules.AddCommand(NewWafRulesUpgradeCmd()) - cmdWafRules.AddCommand(NewWafRulesInspectCmd()) - cmdWafRules.AddCommand(NewWafRulesListCmd()) + cmdWaapRules.AddCommand(NewCmdWaapRulesInstall()) + cmdWaapRules.AddCommand(NewCmdWaapRulesRemove()) + cmdWaapRules.AddCommand(NewCmdWaapRulesUpgrade()) + cmdWaapRules.AddCommand(NewCmdWaapRulesInspect()) + cmdWaapRules.AddCommand(NewCmdWaapRulesList()) - return cmdWafRules + return cmdWaapRules } -func NewWafRulesInstallCmd() *cobra.Command { - var ignoreError bool +func runWaapRulesInstall(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() - var cmdWafRulesInstall = &cobra.Command{ - Use: "install [config]", - Short: "Install given waap-rule(s)", - Long: `Fetch and install given waap-rule(s) from hub`, - Example: `cscli waap-rules install crowdsec/xxx crowdsec/xyz`, + downloadOnly, err := flags.GetBool("download-only") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + ignoreError, err := flags.GetBool("ignore") + if err != nil { + return err + } + + for _, name := range args { + t := cwhub.GetItem(cwhub.WAAP_RULES, name) + if t == nil { + nearestItem, score := GetDistance(cwhub.WAAP_RULES, name) + Suggest(cwhub.WAAP_RULES, name, nearestItem.Name, score, ignoreError) + + continue + } + + if err := cwhub.InstallItem(csConfig, name, cwhub.WAAP_RULES, force, downloadOnly); err != nil { + if !ignoreError { + return fmt.Errorf("error while installing '%s': %w", name, err) + } + log.Errorf("Error while installing '%s': %s", name, err) + } + } + + return nil +} + +func NewCmdWaapRulesInstall() *cobra.Command { + cmdWaapRulesInstall := &cobra.Command{ + Use: "install ...", + Short: "Install given waap rule(s)", + Long: `Fetch and install one or more waap rules from the hub`, + Example: `cscli waap-rules install crowdsecurity/crs`, Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compAllItems(cwhub.WAAP_RULES, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - for _, name := range args { - t := cwhub.GetItem(cwhub.WAAP_RULES, name) - if t == nil { - nearestItem, score := GetDistance(cwhub.WAAP_RULES, name) - Suggest(cwhub.WAAP_RULES, name, nearestItem.Name, score, ignoreError) - continue - } - if err := cwhub.InstallItem(csConfig, name, cwhub.WAAP_RULES, forceAction, downloadOnly); err != nil { - if ignoreError { - log.Errorf("Error while installing '%s': %s", name, err) - } else { - log.Fatalf("Error while installing '%s': %s", name, err) - } - } - } - }, + RunE: runWaapRulesInstall, } - cmdWafRulesInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable") - cmdWafRulesInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files") - cmdWafRulesInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple waf rules") - return cmdWafRulesInstall + flags := cmdWaapRulesInstall.Flags() + flags.BoolP("download-only", "d", false, "Only download packages, don't enable") + flags.Bool("force", false, "Force install: overwrite tainted and outdated files") + flags.Bool("ignore", false, "Ignore errors when installing multiple waap rules") + + return cmdWaapRulesInstall } -func NewWafRulesRemoveCmd() *cobra.Command { - var cmdWafRulesRemove = &cobra.Command{ - Use: "remove [config]", - Short: "Remove given waf-rule(s)", - Long: `Remove given waf-rule(s) from hub`, +func runWaapRulesRemove(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + purge, err := flags.GetBool("purge") + if err != nil { + return err + } + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + err := cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, "", all, purge, force) + if err != nil { + return err + } + + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one waap rule to remove or '--all'") + } + + for _, name := range args { + err := cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, name, all, purge, force) + if err != nil { + return err + } + } + + return nil +} + +func NewCmdWaapRulesRemove() *cobra.Command { + cmdWaapRulesRemove := &cobra.Command{ + Use: "remove ...", + Short: "Remove given waap rule(s)", + Long: `remove one or more waap rules`, + Example: `cscli waap-rules remove crowdsecurity/crs`, Aliases: []string{"delete"}, - Example: `cscli waap-rules remove crowdsec/xxx crowdsec/xyz`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - if all { - cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, "", all, purge, forceAction) - return - } - - if len(args) == 0 { - log.Fatalf("Specify at least one waf rule to remove or '--all' flag.") - } - - for _, name := range args { - cwhub.RemoveMany(csConfig, cwhub.WAAP_RULES, name, all, purge, forceAction) - } - }, + RunE: runWaapRulesRemove, } - cmdWafRulesRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too") - cmdWafRulesRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files") - cmdWafRulesRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the waf rules") - return cmdWafRulesRemove + flags := cmdWaapRulesRemove.Flags() + flags.Bool("purge", false, "Delete source file too") + flags.Bool("force", false, "Force remove: remove tainted and outdated files") + flags.Bool("all", false, "Remove all the waap rules") + + return cmdWaapRulesRemove } -func NewWafRulesUpgradeCmd() *cobra.Command { - var cmdWafRulesUpgrade = &cobra.Command{ - Use: "upgrade [config]", - Short: "Upgrade given waf-rule(s)", - Long: `Fetch and upgrade given waf-rule(s) from hub`, - Example: `cscli waap-rules upgrade crowdsec/xxx crowdsec/xyz`, +func runWaapRulesUpgrade(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + force, err := flags.GetBool("force") + if err != nil { + return err + } + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if all { + if err := cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, "", force); err != nil { + return err + } + return nil + } + + if len(args) == 0 { + return fmt.Errorf("specify at least one waap rule to upgrade or '--all'") + } + + for _, name := range args { + if err := cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, name, force); err != nil { + return err + } + } + + return nil +} + +func NewCmdWaapRulesUpgrade() *cobra.Command { + cmdWaapRulesUpgrade := &cobra.Command{ + Use: "upgrade ...", + Short: "Upgrade given waap rule(s)", + Long: `Fetch and upgrade one or more waap rules from the hub`, + Example: `cscli waap-rules upgrade crowdsecurity/crs`, DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - if all { - cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, "", forceAction) - } else { - if len(args) == 0 { - log.Fatalf("no target waf rule to upgrade") - } - for _, name := range args { - cwhub.UpgradeConfig(csConfig, cwhub.WAAP_RULES, name, forceAction) - } - } - }, + RunE: runWaapRulesUpgrade, } - cmdWafRulesUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the waf rules") - cmdWafRulesUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files") - return cmdWafRulesUpgrade + flags := cmdWaapRulesUpgrade.Flags() + flags.BoolP("all", "a", false, "Upgrade all the waap rules") + flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files") + + return cmdWaapRulesUpgrade } -func NewWafRulesInspectCmd() *cobra.Command { - var cmdWafRulesInspect = &cobra.Command{ - Use: "inspect [name]", - Short: "Inspect given waf rule", - Long: `Inspect given waf rule`, - Example: `cscli waap-rules inspect crowdsec/xxx`, - DisableAutoGenTag: true, +func runWaapRulesInspect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + url, err := flags.GetString("url") + if err != nil { + return err + } + + if url != "" { + csConfig.Cscli.PrometheusUrl = url + } + + noMetrics, err := flags.GetBool("no-metrics") + if err != nil { + return err + } + + for _, name := range args { + if err = InspectItem(name, cwhub.WAAP_RULES, noMetrics); err != nil { + return err + } + } + + return nil +} + +func NewCmdWaapRulesInspect() *cobra.Command { + cmdWaapRulesInspect := &cobra.Command{ + Use: "inspect ", + Short: "Inspect a waap rule", + Long: `Inspect a waap rule`, + Example: `cscli waap-rules inspect crowdsecurity/crs`, Args: cobra.MinimumNArgs(1), + DisableAutoGenTag: true, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return compInstalledItems(cwhub.WAAP_RULES, args, toComplete) }, - Run: func(cmd *cobra.Command, args []string) { - InspectItem(args[0], cwhub.WAAP_RULES) - }, + RunE: runWaapRulesInspect, } - cmdWafRulesInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url") - return cmdWafRulesInspect + flags := cmdWaapRulesInspect.Flags() + flags.StringP("url", "u", "", "Prometheus url") + flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)") + + return cmdWaapRulesInspect } -func NewWafRulesListCmd() *cobra.Command { - var cmdWafRulesList = &cobra.Command{ - Use: "list [name]", - Short: "List all waf rules or given one", - Long: `List all waf rules or given one`, +func runWaapRulesList(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + all, err := flags.GetBool("all") + if err != nil { + return err + } + + if err = ListItems(color.Output, []string{cwhub.WAAP_RULES}, args, false, true, all); err != nil { + return err + } + + return nil +} + +func NewCmdWaapRulesList() *cobra.Command { + cmdWaapRulesList := &cobra.Command{ + Use: "list [waap-rule]...", + Short: "List waap rules", + Long: `List of installed/available/specified waap rules`, Example: `cscli waap-rules list -cscli waap-rules list crowdsecurity/xxx`, +cscli waap-rules list -a +cscli waap-rules list crowdsecurity/crs`, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - ListItems(color.Output, []string{cwhub.WAAP_RULES}, args, false, true, all) - }, + RunE: runWaapRulesList, } - cmdWafRulesList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well") - return cmdWafRulesList + flags := cmdWaapRulesList.Flags() + flags.BoolP("all", "a", false, "List disabled items as well") + + return cmdWaapRulesList } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index c604e670a..e93dbef04 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level { func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) { cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet) if err != nil { - return nil, err - } - - if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) { - return nil, fmt.Errorf("unable to load configuration: common section is empty") + return nil, fmt.Errorf("while loading configuration file: %w", err) } cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags) @@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo dumpStates = true } - // Configuration paths are dependency to load crowdsec configuration - if err := cConfig.LoadConfigurationPaths(); err != nil { - return nil, err - } - if flags.SingleFileType != "" && flags.OneShotDSN != "" { // if we're in time-machine mode, we don't want to log to file cConfig.Common.LogMedia = "stdout" diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 2c1b2970b..42530148c 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -152,14 +152,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { if !config.Enabled { return } - if config.ListenAddr == "" { - log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'") - config.ListenAddr = "127.0.0.1" - } - if config.ListenPort == 0 { - log.Warning("prometheus is enabled, but the listen port is empty, using '6060'") - config.ListenPort = 6060 - } // Registering prometheus // If in aggregated mode, do not register events associated with a source, to keep the cardinality low diff --git a/config/config.yaml b/config/config.yaml index 232b0bc43..2b0e4dfca 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -6,7 +6,6 @@ common: log_max_size: 20 compress_logs: true log_max_files: 10 - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data/ diff --git a/config/config_win.yaml b/config/config_win.yaml index 7863f4fdd..5c34c69a2 100644 --- a/config/config_win.yaml +++ b/config/config_win.yaml @@ -3,7 +3,6 @@ common: log_media: file log_level: info log_dir: C:\ProgramData\CrowdSec\log\ - working_dir: . config_paths: config_dir: C:\ProgramData\CrowdSec\config\ data_dir: C:\ProgramData\CrowdSec\data\ diff --git a/config/config_win_no_lapi.yaml b/config/config_win_no_lapi.yaml index 35c7f2c6f..af240228b 100644 --- a/config/config_win_no_lapi.yaml +++ b/config/config_win_no_lapi.yaml @@ -3,7 +3,6 @@ common: log_media: file log_level: info log_dir: C:\ProgramData\CrowdSec\log\ - working_dir: . config_paths: config_dir: C:\ProgramData\CrowdSec\config\ data_dir: C:\ProgramData\CrowdSec\data\ diff --git a/config/dev.yaml b/config/dev.yaml index 2ff625060..2123dc858 100644 --- a/config/dev.yaml +++ b/config/dev.yaml @@ -2,7 +2,6 @@ common: daemonize: true log_media: stdout log_level: info - working_dir: . config_paths: config_dir: ./config data_dir: ./data/ diff --git a/config/user.yaml b/config/user.yaml index 67bdfa3fc..a1047dcd0 100644 --- a/config/user.yaml +++ b/config/user.yaml @@ -3,7 +3,6 @@ common: log_media: stdout log_level: info log_dir: /var/log/ - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data diff --git a/docker/config.yaml b/docker/config.yaml index 5259a0fe2..681132909 100644 --- a/docker/config.yaml +++ b/docker/config.yaml @@ -3,7 +3,6 @@ common: log_media: stdout log_level: info log_dir: /var/log/ - working_dir: . config_paths: config_dir: /etc/crowdsec/ data_dir: /var/lib/crowdsec/data/ diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index bbe2e1622..c1577782f 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error { log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs)) } - if err := c.LoadCommon(); err != nil { - return fmt.Errorf("loading common configuration: %s", err) - } - c.API.Server.LogDir = c.Common.LogDir c.API.Server.LogMedia = c.Common.LogMedia c.API.Server.CompressLogs = c.Common.CompressLogs diff --git a/pkg/csconfig/api_test.go b/pkg/csconfig/api_test.go index 4338de9c1..10128b76b 100644 --- a/pkg/csconfig/api_test.go +++ b/pkg/csconfig/api_test.go @@ -3,7 +3,6 @@ package csconfig import ( "net" "os" - "path/filepath" "strings" "testing" @@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) { err := tmpLAPI.LoadProfiles() require.NoError(t, err) - LogDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - logLevel := log.InfoLevel config := &Config{} fcontent, err := os.ReadFile("./testdata/config.yaml") @@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) { DbPath: "./testdata/test.db", }, Common: &CommonCfg{ - LogDir: "./testdata/", + LogDir: "./testdata", LogMedia: "stdout", }, DisableAPI: false, @@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) { ShareContext: ptr.Of(false), ConsoleManagement: ptr.Of(false), }, - LogDir: LogDirFullPath, + LogDir: "./testdata", LogMedia: "stdout", OnlineClient: &OnlineApiClientCfg{ CredentialsFilePath: "./testdata/online-api-secrets.yaml", diff --git a/pkg/csconfig/common.go b/pkg/csconfig/common.go index 9d80cd95a..7e1ef6e5c 100644 --- a/pkg/csconfig/common.go +++ b/pkg/csconfig/common.go @@ -14,7 +14,7 @@ type CommonCfg struct { LogMedia string `yaml:"log_media"` LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file LogLevel *log.Level `yaml:"log_level"` - WorkingDir string `yaml:"working_dir,omitempty"` ///var/run + WorkingDir string `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later CompressLogs *bool `yaml:"compress_logs,omitempty"` LogMaxSize int `yaml:"log_max_size,omitempty"` LogMaxAge int `yaml:"log_max_age,omitempty"` @@ -22,15 +22,18 @@ type CommonCfg struct { ForceColorLogs bool `yaml:"force_color_logs,omitempty"` } -func (c *Config) LoadCommon() error { +func (c *Config) loadCommon() error { var err error if c.Common == nil { - return fmt.Errorf("no common block provided in configuration file") + c.Common = &CommonCfg{} + } + + if c.Common.LogMedia == "" { + c.Common.LogMedia = "stdout" } var CommonCleanup = []*string{ &c.Common.LogDir, - &c.Common.WorkingDir, } for _, k := range CommonCleanup { if *k == "" { diff --git a/pkg/csconfig/common_test.go b/pkg/csconfig/common_test.go deleted file mode 100644 index 2c5f798a6..000000000 --- a/pkg/csconfig/common_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package csconfig - -import ( - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/cstest" -) - -func TestLoadCommon(t *testing.T) { - pidDirPath := "./testdata" - LogDirFullPath, err := filepath.Abs("./testdata/log/") - require.NoError(t, err) - - WorkingDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - tests := []struct { - name string - input *Config - expected *CommonCfg - expectedErr string - }{ - { - name: "basic valid configuration", - input: &Config{ - Common: &CommonCfg{ - Daemonize: true, - PidDir: "./testdata", - LogMedia: "file", - LogDir: "./testdata/log/", - WorkingDir: "./testdata/", - }, - }, - expected: &CommonCfg{ - Daemonize: true, - PidDir: pidDirPath, - LogMedia: "file", - LogDir: LogDirFullPath, - WorkingDir: WorkingDirFullPath, - }, - }, - { - name: "empty working dir", - input: &Config{ - Common: &CommonCfg{ - Daemonize: true, - PidDir: "./testdata", - LogMedia: "file", - LogDir: "./testdata/log/", - }, - }, - expected: &CommonCfg{ - Daemonize: true, - PidDir: pidDirPath, - LogMedia: "file", - LogDir: LogDirFullPath, - }, - }, - { - name: "no common", - input: &Config{}, - expected: nil, - expectedErr: "no common block provided in configuration file", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadCommon() - cstest.RequireErrorContains(t, err, tc.expectedErr) - if tc.expectedErr != "" { - return - } - - assert.Equal(t, tc.expected, tc.input.Common) - }) - } -} diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 0fa0e1d2d..4a544729a 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -36,7 +36,7 @@ type Config struct { PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"` DisableAPI bool `yaml:"-"` DisableAgent bool `yaml:"-"` - Hub *Hub `yaml:"-"` + Hub *HubCfg `yaml:"-"` } func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) { @@ -58,18 +58,49 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool // this is actually the "merged" yaml return nil, "", fmt.Errorf("%s: %w", configFile, err) } + + if cfg.Prometheus == nil { + cfg.Prometheus = &PrometheusCfg{} + } + + if cfg.Prometheus.ListenAddr == "" { + cfg.Prometheus.ListenAddr = "127.0.0.1" + log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr) + } + + if cfg.Prometheus.ListenPort == 0 { + cfg.Prometheus.ListenPort = 6060 + log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort) + } + + if err = cfg.loadCommon(); err != nil { + return nil, "", err + } + + if err = cfg.loadConfigurationPaths(); err != nil { + return nil, "", err + } + + if err = cfg.loadHub(); err != nil { + return nil, "", err + } + + if err = cfg.loadCSCLI(); err != nil { + return nil, "", err + } + return &cfg, configData, nil } +// XXX: We must not have a different behavior with an empty vs a missing configuration file. +// XXX: For this reason, all defaults have to come from NewConfig(). The following function should +// XXX: be replaced func NewDefaultConfig() *Config { logLevel := log.InfoLevel commonCfg := CommonCfg{ Daemonize: false, - PidDir: "/tmp/", LogMedia: "stdout", - //LogDir unneeded LogLevel: &logLevel, - WorkingDir: ".", } prometheus := PrometheusCfg{ Enabled: true, diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 24ff454b7..07db4bd71 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -15,21 +15,25 @@ type ConfigurationPaths struct { NotificationDir string `yaml:"notification_dir,omitempty"` } -func (c *Config) LoadConfigurationPaths() error { +func (c *Config) loadConfigurationPaths() error { var err error if c.ConfigPaths == nil { + // XXX: test me return fmt.Errorf("no configuration paths provided") } if c.ConfigPaths.DataDir == "" { + // XXX: test me return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section") } if c.ConfigPaths.HubDir == "" { + // XXX: test me c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub") } if c.ConfigPaths.HubIndexFile == "" { + // XXX: test me c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json") } diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index 085c873a5..ab391eaa9 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -149,10 +149,6 @@ func (c *Config) LoadCrowdsec() error { return fmt.Errorf("loading api client: %s", err) } - if err := c.LoadHub(); err != nil { - return fmt.Errorf("while loading hub: %w", err) - } - c.Crowdsec.ContextToSend = make(map[string][]string, 0) fallback := false if c.Crowdsec.ConsoleContextPath == "" { diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index aa1d341f5..06a7e91bd 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) { acquisDirFullPath, err := filepath.Abs("./testdata/acquis") require.NoError(t, err) - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - contextFileFullPath, err := filepath.Abs("./testdata/context.yaml") require.NoError(t, err) @@ -66,10 +54,11 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: "", ConsoleContextPath: contextFileFullPath, AcquisitionFilePath: acquisFullPath, - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, + ConfigDir: "./testdata", + DataDir: "./data", + HubDir: "./hub", + // XXX: need to ensure a default here + HubIndexFile: "", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -109,10 +98,11 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: acquisDirFullPath, AcquisitionFilePath: acquisFullPath, ConsoleContextPath: contextFileFullPath, - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, + ConfigDir: "./testdata", + // XXX: need to ensure a default here + HubIndexFile: "", + DataDir: "./data", + HubDir: "./hub", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -141,7 +131,7 @@ func TestLoadCrowdsec(t *testing.T) { }, }, Crowdsec: &CrowdsecServiceCfg{ - ConsoleContextPath: contextFileFullPath, + ConsoleContextPath: "./testdata/context.yaml", ConsoleContextValueLength: 10, }, }, @@ -149,10 +139,11 @@ func TestLoadCrowdsec(t *testing.T) { Enable: ptr.Of(true), AcquisitionDirPath: "", AcquisitionFilePath: "", - ConfigDir: configDirFullPath, - HubIndexFile: hubIndexFileFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, + ConfigDir: "./testdata", + // XXX: need to ensure a default here + HubIndexFile: "", + DataDir: "./data", + HubDir: "./hub", ConsoleContextPath: contextFileFullPath, BucketsRoutinesCount: 1, ParserRoutinesCount: 1, diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index 6b0bf5ae4..8db0a1848 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -1,5 +1,9 @@ package csconfig +import ( + "fmt" +) + /*cscli specific config, such as hub directory*/ type CscliCfg struct { Output string `yaml:"output,omitempty"` @@ -15,17 +19,18 @@ type CscliCfg struct { PrometheusUrl string `yaml:"prometheus_uri"` } -func (c *Config) LoadCSCLI() error { +func (c *Config) loadCSCLI() error { if c.Cscli == nil { c.Cscli = &CscliCfg{} } - if err := c.LoadConfigurationPaths(); err != nil { - return err - } c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir c.Cscli.DataDir = c.ConfigPaths.DataDir c.Cscli.HubDir = c.ConfigPaths.HubDir c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile + if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { + c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) + } + return nil } diff --git a/pkg/csconfig/cscli_test.go b/pkg/csconfig/cscli_test.go index b3d0abc6b..e928eda5a 100644 --- a/pkg/csconfig/cscli_test.go +++ b/pkg/csconfig/cscli_test.go @@ -1,28 +1,14 @@ package csconfig import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestLoadCSCLI(t *testing.T) { - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - tests := []struct { name string input *Config @@ -38,26 +24,27 @@ func TestLoadCSCLI(t *testing.T) { HubDir: "./hub", HubIndexFile: "./hub/.index.json", }, + Prometheus: &PrometheusCfg{ + Enabled: true, + Level: "full", + ListenAddr: "127.0.0.1", + ListenPort: 6060, + }, }, expected: &CscliCfg{ - ConfigDir: configDirFullPath, - DataDir: dataFullPath, - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, + ConfigDir: "./testdata", + DataDir: "./data", + HubDir: "./hub", + HubIndexFile: "./hub/.index.json", + PrometheusUrl: "http://127.0.0.1:6060/metrics", }, }, - { - name: "no configuration path", - input: &Config{}, - expected: &CscliCfg{}, - expectedErr: "no configuration paths provided", - }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadCSCLI() + err := tc.input.loadCSCLI() cstest.RequireErrorContains(t, err, tc.expectedErr) if tc.expectedErr != "" { return diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index 4c3c610aa..2164c19b2 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -1,19 +1,15 @@ package csconfig -/*cscli specific config, such as hub directory*/ -type Hub struct { - HubIndexFile string - HubDir string - InstallDir string - InstallDataDir string +// HubConfig holds the configuration for a hub +type HubCfg struct { + HubIndexFile string // Path to the local index file + HubDir string // Where the hub items are downloaded + InstallDir string // Where to install items + InstallDataDir string // Where to install data } -func (c *Config) LoadHub() error { - if err := c.LoadConfigurationPaths(); err != nil { - return err - } - - c.Hub = &Hub{ +func (c *Config) loadHub() error { + c.Hub = &HubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, InstallDir: c.ConfigPaths.ConfigDir, diff --git a/pkg/csconfig/hub_test.go b/pkg/csconfig/hub_test.go index d573e4690..0fa627ae0 100644 --- a/pkg/csconfig/hub_test.go +++ b/pkg/csconfig/hub_test.go @@ -1,32 +1,18 @@ package csconfig import ( - "path/filepath" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestLoadHub(t *testing.T) { - hubFullPath, err := filepath.Abs("./hub") - require.NoError(t, err) - - dataFullPath, err := filepath.Abs("./data") - require.NoError(t, err) - - configDirFullPath, err := filepath.Abs("./testdata") - require.NoError(t, err) - - hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json") - require.NoError(t, err) - tests := []struct { name string input *Config - expected *Hub + expected *HubCfg expectedErr string }{ { @@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) { HubIndexFile: "./hub/.index.json", }, }, - expected: &Hub{ - HubDir: hubFullPath, - HubIndexFile: hubIndexFileFullPath, - InstallDir: configDirFullPath, - InstallDataDir: dataFullPath, + expected: &HubCfg{ + HubDir: "./hub", + HubIndexFile: "./hub/.index.json", + InstallDir: "./testdata", + InstallDataDir: "./data", }, }, - { - name: "no data dir", - input: &Config{ - ConfigPaths: &ConfigurationPaths{ - ConfigDir: "./testdata", - HubDir: "./hub", - HubIndexFile: "./hub/.index.json", - }, - }, - expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section", - }, - { - name: "no configuration path", - input: &Config{}, - expectedErr: "no configuration paths provided", - }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadHub() + err := tc.input.loadHub() cstest.RequireErrorContains(t, err, tc.expectedErr) if tc.expectedErr != "" { return diff --git a/pkg/csconfig/prometheus.go b/pkg/csconfig/prometheus.go index eea768ab7..9b80fe398 100644 --- a/pkg/csconfig/prometheus.go +++ b/pkg/csconfig/prometheus.go @@ -1,19 +1,8 @@ package csconfig -import "fmt" - type PrometheusCfg struct { Enabled bool `yaml:"enabled"` Level string `yaml:"level"` //aggregated|full ListenAddr string `yaml:"listen_addr"` ListenPort int `yaml:"listen_port"` } - -func (c *Config) LoadPrometheus() error { - if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil { - if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { - c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) - } - } - return nil -} diff --git a/pkg/csconfig/prometheus_test.go b/pkg/csconfig/prometheus_test.go deleted file mode 100644 index 79c9ec58f..000000000 --- a/pkg/csconfig/prometheus_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package csconfig - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/crowdsecurity/go-cs-lib/cstest" -) - -func TestLoadPrometheus(t *testing.T) { - tests := []struct { - name string - input *Config - expectedURL string - expectedErr string - }{ - { - name: "basic valid configuration", - input: &Config{ - Prometheus: &PrometheusCfg{ - Enabled: true, - Level: "full", - ListenAddr: "127.0.0.1", - ListenPort: 6060, - }, - Cscli: &CscliCfg{}, - }, - expectedURL: "http://127.0.0.1:6060", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - err := tc.input.LoadPrometheus() - cstest.RequireErrorContains(t, err, tc.expectedErr) - - require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl) - }) - } -} diff --git a/pkg/csconfig/simulation.go b/pkg/csconfig/simulation.go index 184708f0d..0d09aa478 100644 --- a/pkg/csconfig/simulation.go +++ b/pkg/csconfig/simulation.go @@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool { } func (c *Config) LoadSimulation() error { - - if err := c.LoadConfigurationPaths(); err != nil { - return err - } - simCfg := SimulationConfig{} if c.ConfigPaths.SimulationFilePath == "" { c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml") diff --git a/pkg/csconfig/simulation_test.go b/pkg/csconfig/simulation_test.go index 44b8909a2..01f05e397 100644 --- a/pkg/csconfig/simulation_test.go +++ b/pkg/csconfig/simulation_test.go @@ -2,7 +2,6 @@ package csconfig import ( "fmt" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -12,12 +11,6 @@ import ( ) func TestSimulationLoading(t *testing.T) { - testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml") - require.NoError(t, err) - - badYamlFullPath, err := filepath.Abs("./testdata/config.yaml") - require.NoError(t, err) - tests := []struct { name string input *Config @@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage), + expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage), }, { name: "basic bad file content", @@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), + expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors", }, { name: "basic bad file content", @@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) { }, Crowdsec: &CrowdsecServiceCfg{}, }, - expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), + expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors", }, } diff --git a/pkg/csconfig/testdata/config.yaml b/pkg/csconfig/testdata/config.yaml index 288c09b84..17975b105 100644 --- a/pkg/csconfig/testdata/config.yaml +++ b/pkg/csconfig/testdata/config.yaml @@ -2,7 +2,6 @@ common: daemonize: false log_media: stdout log_level: info - working_dir: . prometheus: enabled: true level: full diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index e9d3fb68f..ee0e3bf55 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -1,66 +1,45 @@ +// Package cwhub is responsible for installing and upgrading the local hub files. +// +// This includes retrieving the index, the items to install (parsers, scenarios, data files...) +// and managing the dependencies and taints. package cwhub import ( "fmt" "os" "path/filepath" - "sort" "strings" "github.com/enescakir/emoji" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" ) -const ( - HubIndexFile = ".index.json" - - // managed item types - PARSERS = "parsers" - PARSERS_OVFLW = "postoverflows" - SCENARIOS = "scenarios" - COLLECTIONS = "collections" - WAAP_RULES = "waap-rules" -) - var ( - ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS, WAAP_RULES} - ErrMissingReference = errors.New("Reference(s) missing in collection") - // XXX: can we remove these globals? - skippedLocal = 0 - skippedTainted = 0 RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" HubBranch = "master" - hubIdx map[string]map[string]Item ) +// ItemVersion is used to detect the version of a given item +// by comparing the hash of each version to the local file. +// If the item does not match any known version, it is considered tainted. type ItemVersion struct { - Digest string `json:"digest,omitempty"` // meow - Deprecated bool `json:"deprecated,omitempty"` + Digest string `json:"digest,omitempty"` // meow + Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this? } -type ItemHubStatus struct { - Name string `json:"name"` - LocalVersion string `json:"local_version"` - LocalPath string `json:"local_path"` - Description string `json:"description"` - UTF8Status string `json:"utf8_status"` - Status string `json:"status"` -} - -// Item can be: parser, scenario, collection.. +// Item represents an object managed in the hub. It can be a parser, scenario, collection.. type Item struct { // descriptive info Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich) Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-... - Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name" + Name string `json:"name,omitempty"` // as seen in .index.json, usually "author/name" FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml - Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .config.json - Author string `json:"author,omitempty"` // as seen in .config.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .config.json + Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json + Author string `json:"author,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any // remote (hub) info @@ -78,7 +57,7 @@ type Item struct { Tainted bool `json:"tainted,omitempty"` // has it been locally modified Local bool `json:"local,omitempty"` // if it's a non versioned control one - // if it's a collection, it's not a single file + // if it's a collection, it can have sub items Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"` PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"` Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"` @@ -86,7 +65,9 @@ type Item struct { WafRules []string `json:"waap-rules,omitempty" yaml:"waap-rules,omitempty"` } -func (i *Item) status() (string, emoji.Emoji) { +// Status returns the status of the item as a string and an emoji +// ie. "enabled,update-available" and emoji.Warning +func (i *Item) Status() (string, emoji.Emoji) { status := "disabled" ok := false @@ -126,26 +107,14 @@ func (i *Item) status() (string, emoji.Emoji) { return status, emo } -func (i *Item) hubStatus() ItemHubStatus { - status, emo := i.status() - - return ItemHubStatus{ - Name: i.Name, - LocalVersion: i.LocalVersion, - LocalPath: i.LocalPath, - Description: i.Description, - Status: status, - UTF8Status: fmt.Sprintf("%v %s", emo, status), - } -} - // versionStatus: semver requires 'v' prefix func (i *Item) versionStatus() int { return semver.Compare("v"+i.Version, "v"+i.LocalVersion) } +// GetItemMap returns the map of items for a given type func GetItemMap(itemType string) map[string]Item { - m, ok := hubIdx[itemType] + m, ok := hubIdx.Items[itemType] if !ok { return nil } @@ -153,7 +122,7 @@ func GetItemMap(itemType string) map[string]Item { return m } -// Given a FileInfo, extract the map key. Follow a symlink if necessary +// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary func itemKey(itemPath string) (string, error) { f, err := os.Lstat(itemPath) if err != nil { @@ -200,6 +169,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { return &v, nil } +// GetItem returns the item from hub based on its type and full name (author/name) func GetItem(itemType string, itemName string) *Item { if m, ok := GetItemMap(itemType)[itemName]; ok { return &m @@ -208,10 +178,28 @@ func GetItem(itemType string, itemName string) *Item { return nil } +// GetItemNames returns the list of item (full) names for a given type +// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx +// The names can be used to retrieve the item with GetItem() +func GetItemNames(itemType string) []string { + m := GetItemMap(itemType) + if m == nil { + return nil + } + + names := make([]string, 0, len(m)) + for k := range m { + names = append(names, k) + } + + return names +} + +// AddItem adds an item to the hub index func AddItem(itemType string, item Item) error { for _, itype := range ItemTypes { if itype == itemType { - hubIdx[itemType][item.Name] = item + hubIdx.Items[itemType][item.Name] = item return nil } } @@ -219,17 +207,9 @@ func AddItem(itemType string, item Item) error { return fmt.Errorf("ItemType %s is unknown", itemType) } -func DisplaySummary() { - log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers,%d waf rules", len(hubIdx[COLLECTIONS]), - len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[WAAP_RULES])) - - if skippedLocal > 0 || skippedTainted > 0 { - log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted) - } -} - +// GetInstalledItems returns the list of installed items func GetInstalledItems(itemType string) ([]Item, error) { - items, ok := hubIdx[itemType] + items, ok := hubIdx.Items[itemType] if !ok { return nil, fmt.Errorf("no %s in hubIdx", itemType) } @@ -245,6 +225,7 @@ func GetInstalledItems(itemType string) ([]Item, error) { return retItems, nil } +// GetInstalledItemsAsString returns the names of the installed items func GetInstalledItemsAsString(itemType string) ([]string, error) { items, err := GetInstalledItems(itemType) if err != nil { @@ -259,32 +240,3 @@ func GetInstalledItemsAsString(itemType string) ([]string, error) { return retStr, nil } - -// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy) -func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus { - if _, ok := hubIdx[itemType]; !ok { - log.Errorf("type %s doesn't exist", itemType) - - return nil - } - - ret := make([]ItemHubStatus, 0) - - // remember, you do it for the user :) - for _, item := range hubIdx[itemType] { - if name != "" && name != item.Name { - // user has requested a specific name - continue - } - // Only enabled items ? - if !all && !item.Installed { - continue - } - // Check the item status - ret = append(ret, item.hubStatus()) - } - - sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name }) - - return ret -} diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 2fef828c2..7f3a50a1a 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -53,7 +53,7 @@ func TestItemStatus(t *testing.T) { item.Local = false item.Tainted = false - txt, _ := item.status() + txt, _ := item.Status() require.Equal(t, "enabled,update-available", txt) item.Installed = false @@ -61,7 +61,7 @@ func TestItemStatus(t *testing.T) { item.Local = true item.Tainted = false - txt, _ = item.status() + txt, _ = item.Status() require.Equal(t, "disabled,local", txt) } @@ -121,7 +121,7 @@ func TestIndexDownload(t *testing.T) { } func getTestCfg() *csconfig.Config { - cfg := &csconfig.Config{Hub: &csconfig.Hub{}} + cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}} cfg.Hub.InstallDir, _ = filepath.Abs("./install") cfg.Hub.HubDir, _ = filepath.Abs("./hubdir") cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json") @@ -172,7 +172,7 @@ func envTearDown(cfg *csconfig.Config) { } } -func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { +func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { // Install the parser err := DownloadLatest(cfg, &item, false, false) require.NoError(t, err, "failed to download %s", item.Name) @@ -180,9 +180,9 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) err = EnableItem(cfg, &item) require.NoError(t, err, "failed to enable %s", item.Name) @@ -190,11 +190,11 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) } -func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) +func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) @@ -208,11 +208,11 @@ func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) } -func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) +func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { + assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status err := DownloadLatest(cfg, &item, true, true) @@ -222,12 +222,12 @@ func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) { _, err = LocalSync(cfg) require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) } -func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { - assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name) +func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { + assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) // Remove err := DisableItem(cfg, &item, false, false) @@ -238,9 +238,9 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) // Purge err = DisableItem(cfg, &item, true, false) @@ -251,8 +251,8 @@ func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) { require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) } func TestInstallParser(t *testing.T) { @@ -270,17 +270,15 @@ func TestInstallParser(t *testing.T) { getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx[PARSERS] { + for _, it := range hubIdx.Items[PARSERS] { testInstallItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) + it = hubIdx.Items[PARSERS][it.Name] testTaintItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] - _ = GetHubStatusForItemType(PARSERS, it.Name, false) + it = hubIdx.Items[PARSERS][it.Name] testUpdateItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] + it = hubIdx.Items[PARSERS][it.Name] testDisableItem(cfg.Hub, t, it) - it = hubIdx[PARSERS][it.Name] + it = hubIdx.Items[PARSERS][it.Name] break } @@ -301,19 +299,14 @@ func TestInstallCollection(t *testing.T) { getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx[COLLECTIONS] { + for _, it := range hubIdx.Items[COLLECTIONS] { testInstallItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] + it = hubIdx.Items[COLLECTIONS][it.Name] testTaintItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] + it = hubIdx.Items[COLLECTIONS][it.Name] testUpdateItem(cfg.Hub, t, it) - it = hubIdx[COLLECTIONS][it.Name] + it = hubIdx.Items[COLLECTIONS][it.Name] testDisableItem(cfg.Hub, t, it) - - it = hubIdx[COLLECTIONS][it.Name] - x := GetHubStatusForItemType(COLLECTIONS, it.Name, false) - log.Infof("%+v", x) - break } } diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index ef111ba62..7b6771867 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -19,20 +19,21 @@ import ( var ErrIndexNotFound = fmt.Errorf("index not found") -func UpdateHubIdx(hub *csconfig.Hub) error { +// UpdateHubIdx downloads the latest version of the index and updates the one in memory +func UpdateHubIdx(hub *csconfig.HubCfg) error { bidx, err := DownloadHubIdx(hub) if err != nil { return fmt.Errorf("failed to download index: %w", err) } - ret, err := LoadPkgIndex(bidx) + ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { return fmt.Errorf("failed to read index: %w", err) } } - hubIdx = ret + hubIdx = HubIndex{Items: ret} if _, err := LocalSync(hub); err != nil { return fmt.Errorf("failed to sync: %w", err) @@ -41,7 +42,8 @@ func UpdateHubIdx(hub *csconfig.Hub) error { return nil } -func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { +// DownloadHubIdx downloads the latest version of the index and returns the content +func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) @@ -96,7 +98,7 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) { } // DownloadLatest will download the latest version of Item to the tdir directory -func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error { +func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) @@ -115,7 +117,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx[ptrtype][p] + val, ok := hubIdx.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } @@ -151,7 +153,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly } } - hubIdx[ptrtype][p] = val + hubIdx.Items[ptrtype][p] = val } } @@ -163,7 +165,7 @@ func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly return nil } -func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error { +func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { tdir := hub.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files @@ -265,12 +267,13 @@ func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error { return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) } - hubIdx[target.Type][target.Name] = *target + hubIdx.Items[target.Type][target.Name] = *target return nil } -func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { +// DownloadDataIfNeeded downloads the data files for an item +func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error { itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName) itemFile, err := os.Open(itemFilePath) @@ -287,6 +290,7 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error { return nil } +// downloadData downloads the data files for an item func downloadData(dataFolder string, force bool, reader io.Reader) error { var err error diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index 351b08f8e..b1b41c579 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -17,7 +17,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = "x" - ret, err := DownloadHubIdx(&csconfig.Hub{}) + ret, err := DownloadHubIdx(&csconfig.HubCfg{}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { log.Errorf("unexpected error %s", err) } @@ -29,7 +29,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = "https://baddomain/%s/%s" - ret, err = DownloadHubIdx(&csconfig.Hub{}) + ret, err = DownloadHubIdx(&csconfig.HubCfg{}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { log.Errorf("unexpected error %s", err) } @@ -41,7 +41,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = back - ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"}) + ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"}) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { log.Errorf("unexpected error %s", err) } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index c17e6758d..2db768636 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -12,12 +12,12 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) -// pick a hub branch corresponding to the current crowdsec version. +// chooseHubBranch returns the branch name to use for the hub +// It can be "master" or branch corresponding to the current crowdsec version func chooseHubBranch() string { latest, err := cwversion.Latest() if err != nil { log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err) - //lint:ignore nilerr return "master" } @@ -61,8 +61,9 @@ func SetHubBranch() { log.Debugf("Using branch '%s' for the hub", HubBranch) } -func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error { - item := GetItem(obtype, name) +// InstallItem installs an item from the hub +func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error { + item := GetItem(itemType, name) if item == nil { return fmt.Errorf("unable to retrieve item: %s", name) } @@ -80,7 +81,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return fmt.Errorf("while downloading %s: %w", item.Name, err) } - if err = AddItem(obtype, *item); err != nil { + if err = AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -94,7 +95,7 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return fmt.Errorf("while enabling %s: %w", item.Name, err) } - if err := AddItem(obtype, *item); err != nil { + if err := AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -103,29 +104,29 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo return nil } -// XXX this must return errors instead of log.Fatal -func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) { +// RemoveItem removes one - or all - the items from the hub +func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error { if name != "" { item := GetItem(itemType, name) if item == nil { - log.Fatalf("unable to retrieve: %s", name) + return fmt.Errorf("can't find '%s' in %s", name, itemType) } err := DisableItem(csConfig.Hub, item, purge, forceAction) if err != nil { - log.Fatalf("unable to disable %s : %v", item.Name, err) + return fmt.Errorf("unable to disable %s: %w", item.Name, err) } if err = AddItem(itemType, *item); err != nil { - log.Fatalf("unable to add %s: %v", item.Name, err) + return fmt.Errorf("unable to add %s: %w", item.Name, err) } - return + return nil } if !all { - log.Fatal("removing item: no item specified") + return fmt.Errorf("removing item: no item specified") } disabled := 0 @@ -138,19 +139,22 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo err := DisableItem(csConfig.Hub, &v, purge, forceAction) if err != nil { - log.Fatalf("unable to disable %s : %v", v.Name, err) + return fmt.Errorf("unable to disable %s: %w", v.Name, err) } if err := AddItem(itemType, v); err != nil { - log.Fatalf("unable to add %s: %v", v.Name, err) + return fmt.Errorf("unable to add %s: %w", v.Name, err) } disabled++ } log.Infof("Disabled %d items", disabled) + + return nil } -func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) { +// UpgradeConfig upgrades an item from the hub +func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error { updated := 0 found := false @@ -165,17 +169,17 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } if !v.Downloaded { - log.Warningf("%s : not downloaded, please install.", v.Name) + log.Warningf("%s: not downloaded, please install.", v.Name) continue } found = true if v.UpToDate { - log.Infof("%s : up-to-date", v.Name) + log.Infof("%s: up-to-date", v.Name) if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil { - log.Fatalf("%s : download failed : %v", v.Name, err) + return fmt.Errorf("%s: download failed: %w", v.Name, err) } if !force { @@ -184,7 +188,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil { - log.Fatalf("%s : download failed : %v", v.Name, err) + return fmt.Errorf("%s: download failed: %w", v.Name, err) } if !v.UpToDate { @@ -202,14 +206,14 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } if err := AddItem(itemType, v); err != nil { - log.Fatalf("unable to add %s: %v", v.Name, err) + return fmt.Errorf("unable to add %s: %w", v.Name, err) } } if !found && name == "" { log.Infof("No %s installed, nothing to upgrade", itemType) } else if !found { - log.Errorf("Item '%s' not found in hub", name) + log.Errorf("can't find '%s' in %s", name, itemType) } else if updated == 0 && found { if name == "" { log.Infof("All %s are already up-to-date", itemType) @@ -219,4 +223,6 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } else if updated != 0 { log.Infof("Upgraded %d items", updated) } + + return nil } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index c8bb28c36..ecb778fdc 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -15,19 +15,19 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // fresh install of collection getHubIdxOrFail(t) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) // This is the scenario that gets added in next version of collection - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") @@ -40,16 +40,17 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { getHubIdxOrFail(t) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + require.NoError(t, err) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } // Install a collection, disable a scenario. @@ -61,36 +62,39 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { // fresh install of collection getHubIdxOrFail(t) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + require.NoError(t, err) + getHubIdxOrFail(t) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - if err := UpdateHubIdx(cfg.Hub); err != nil { + if err = UpdateHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to download index : %s", err) } - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + require.NoError(t, err) getHubIdxOrFail(t) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) } func getHubIdxOrFail(t *testing.T) { @@ -109,51 +113,55 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * // fresh install of collection getHubIdxOrFail(t) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + require.NoError(t, err) + getHubIdxOrFail(t) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - if err := UpdateHubIdx(cfg.Hub); err != nil { + if err = UpdateHubIdx(cfg.Hub); err != nil { t.Fatalf("failed to download index : %s", err) } - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) getHubIdxOrFail(t) - UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + require.NoError(t, err) + getHubIdxOrFail(t) - require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } func assertCollectionDepsInstalled(t *testing.T, collection string) { t.Helper() - c := hubIdx[COLLECTIONS][collection] + c := hubIdx.Items[COLLECTIONS][collection] require.NoError(t, CollecDepsCheck(&c)) } diff --git a/pkg/cwhub/hubindex.go b/pkg/cwhub/hubindex.go new file mode 100644 index 000000000..59070d8e3 --- /dev/null +++ b/pkg/cwhub/hubindex.go @@ -0,0 +1,105 @@ +package cwhub + +import ( + "encoding/json" + "fmt" + "strings" + + log "github.com/sirupsen/logrus" +) + +const ( + HubIndexFile = ".index.json" + + // managed item types + COLLECTIONS = "collections" + PARSERS = "parsers" + POSTOVERFLOWS = "postoverflows" + SCENARIOS = "scenarios" + WAAP_RULES = "waap-rules" +) + +var ( + // XXX: The order is important, as it is used to construct the + // index tree in memory --> collections must be last + ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS, WAAP_RULES} + hubIdx = HubIndex{} +) + +type HubItems map[string]map[string]Item + +// HubIndex represents the runtime status of the hub (parsed items, etc.) +// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared +type HubIndex struct { + Items HubItems + skippedLocal int + skippedTainted int +} + +// displaySummary prints a total count of the hub items +func (h HubIndex) displaySummary() { + msg := "Loaded: " + for itemType := range h.Items { + msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) + } + log.Info(strings.Trim(msg, ", ")) + + if h.skippedLocal > 0 || h.skippedTainted > 0 { + log.Infof("unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted) + } +} + +// DisplaySummary prints a total count of the hub items. +// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton +func DisplaySummary() { + hubIdx.displaySummary() +} + +// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections +func ParseIndex(buff []byte) (HubItems, error) { + var ( + RawIndex HubItems + missingItems []string + ) + + if err := json.Unmarshal(buff, &RawIndex); err != nil { + return nil, fmt.Errorf("failed to unmarshal index: %w", err) + } + + log.Debugf("%d item types in hub index", len(ItemTypes)) + + // Iterate over the different types to complete the struct + for _, itemType := range ItemTypes { + log.Tracef("%s: %d items", itemType, len(RawIndex[itemType])) + + for name, item := range RawIndex[itemType] { + item.Name = name + item.Type = itemType + x := strings.Split(item.RemotePath, "/") + item.FileName = x[len(x)-1] + RawIndex[itemType][name] = item + + if itemType != COLLECTIONS { + continue + } + + // if it's a collection, check its sub-items are present + // XXX should be done later + for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} { + ptrtype := ItemTypes[idx] + for _, p := range ptr { + if _, ok := RawIndex[ptrtype][p]; !ok { + log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) + missingItems = append(missingItems, p) + } + } + } + } + } + + if len(missingItems) > 0 { + return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference) + } + + return RawIndex, nil +} diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 45e2ba419..4ac955b7b 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -10,7 +10,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func purgeItem(hub *csconfig.Hub, target Item) (Item, error) { +func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) { itempath := hub.HubDir + "/" + target.RemotePath // disable hub file @@ -20,13 +20,13 @@ func purgeItem(hub *csconfig.Hub, target Item) (Item, error) { target.Downloaded = false log.Infof("Removed source file [%s]: %s", target.Name, itempath) - hubIdx[target.Type][target.Name] = target + hubIdx.Items[target.Type][target.Name] = target return target, nil } // DisableItem to disable an item managed by the hub, removes the symlink if purge is true -func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error { +func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error { var err error // already disabled, noop unless purge @@ -54,7 +54,7 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - if val, ok := hubIdx[ptrtype][p]; ok { + if val, ok := hubIdx.Items[ptrtype][p]; ok { // check if the item doesn't belong to another collection before removing it toRemove := true @@ -130,14 +130,14 @@ func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error } } - hubIdx[target.Type][target.Name] = *target + hubIdx.Items[target.Type][target.Name] = *target return nil } // creates symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively -func EnableItem(hub *csconfig.Hub, target *Item) error { +func EnableItem(hub *csconfig.HubCfg, target *Item) error { var err error parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/") @@ -172,7 +172,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error { for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx[ptrtype][p] + val, ok := hubIdx.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } @@ -208,7 +208,7 @@ func EnableItem(hub *csconfig.Hub, target *Item) error { log.Infof("Enabled %s : %s", target.Type, target.Name) target.Installed = true - hubIdx[target.Type][target.Name] = *target + hubIdx.Items[target.Type][target.Name] = *target return nil } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index dcc19f863..12bd828cf 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -2,7 +2,6 @@ package cwhub import ( "crypto/sha256" - "encoding/json" "errors" "fmt" "io" @@ -67,7 +66,7 @@ type Walker struct { installdir string } -func NewWalker(hub *csconfig.Hub) Walker { +func NewWalker(hub *csconfig.HubCfg) Walker { return Walker{ hubdir: hub.HubDir, installdir: hub.InstallDir, @@ -98,7 +97,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { //.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/profiles/crowdsec/linux.yaml if len(subs) < 4 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) + return itemFileInfo{}, false, fmt.Errorf("path is too short : %s (%d)", path, len(subs)) } ret.fname = subs[len(subs)-1] @@ -108,7 +107,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { } else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec//... log.Tracef("in install dir") if len(subs) < 3 { - log.Fatalf("path is too short : %s (%d)", path, len(subs)) + return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } ///.../config/parser/stage/file.yaml ///.../config/postoverflow/stage/file.yaml @@ -134,7 +133,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { } else if ret.stage == WAAP_RULES { ret.ftype = WAAP_RULES ret.stage = "" - } else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW { + } else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { // its a PARSER / PARSER_OVFLW with a stage return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path) } @@ -144,7 +143,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { return ret, inhub, nil } -func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { +func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { var ( local bool hubpath string @@ -201,12 +200,12 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { // if it's not a symlink and not in hub, it's a local file, don't bother if local && !inhub { log.Tracef("%s is a local file, skip", path) - skippedLocal++ + hubIdx.skippedLocal++ // log.Infof("local scenario, skip.") _, fileName := filepath.Split(path) - hubIdx[info.ftype][info.fname] = Item{ + hubIdx.Items[info.ftype][info.fname] = Item{ Name: info.fname, Stage: info.stage, Installed: true, @@ -225,7 +224,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { match := false - for name, item := range hubIdx[info.ftype] { + for name, item := range hubIdx.Items[info.ftype] { log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") if info.fname != item.FileName { @@ -307,7 +306,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { if !match { log.Tracef("got tainted match for %s: %s", item.Name, path) - skippedTainted++ + hubIdx.skippedTainted++ // the file and the stage is right, but the hash is wrong, it has been tainted by user if !inhub { item.LocalPath = path @@ -320,14 +319,7 @@ func (w Walker) parserVisit(path string, f os.DirEntry, err error) error { item.LocalHash = sha } - // update the entry if appropriate - // if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D { - // fmt.Printf("Updating %s", k) - // hubIdx[ftype][k] = v - // } else if !inhub { - - // } else if - hubIdx[info.ftype][name] = item + hubIdx.Items[info.ftype][name] = item return nil } @@ -353,9 +345,9 @@ func CollecDepsCheck(v *Item) error { for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { sliceType := ItemTypes[idx] for _, subName := range itemSlice { - subItem, ok := hubIdx[sliceType][subName] + subItem, ok := hubIdx.Items[sliceType][subName] if !ok { - log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name) + return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name) } log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed) @@ -375,7 +367,7 @@ func CollecDepsCheck(v *Item) error { return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) } - hubIdx[sliceType][subName] = subItem + hubIdx.Items[sliceType][subName] = subItem } // propagate the state of sub-items to set @@ -406,7 +398,7 @@ func CollecDepsCheck(v *Item) error { subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) } - hubIdx[sliceType][subName] = subItem + hubIdx.Items[sliceType][subName] = subItem log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate) } @@ -415,23 +407,23 @@ func CollecDepsCheck(v *Item) error { return nil } -func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { +func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { warnings := []string{} - // For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last + // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last for _, scan := range ItemTypes { cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { log.Errorf("failed %s : %s", cpath, err) } - err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit) + err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit) if err != nil { return warnings, err } } - for name, item := range hubIdx[COLLECTIONS] { + for name, item := range hubIdx.Items[COLLECTIONS] { if !item.Installed { continue } @@ -441,7 +433,7 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { case 0: // latest if err := CollecDepsCheck(&item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) - hubIdx[COLLECTIONS][name] = item + hubIdx.Items[COLLECTIONS][name] = item } case 1: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) @@ -456,9 +448,9 @@ func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) { } // Updates the info from HubInit() with the local state -func LocalSync(hub *csconfig.Hub) ([]string, error) { - skippedLocal = 0 - skippedTainted = 0 +func LocalSync(hub *csconfig.HubCfg) ([]string, error) { + hubIdx.skippedLocal = 0 + hubIdx.skippedTainted = 0 warnings, err := SyncDir(hub, hub.InstallDir) if err != nil { @@ -473,7 +465,7 @@ func LocalSync(hub *csconfig.Hub) ([]string, error) { return warnings, nil } -func GetHubIdx(hub *csconfig.Hub) error { +func GetHubIdx(hub *csconfig.HubCfg) error { if hub == nil { return fmt.Errorf("no configuration found for hub") } @@ -485,7 +477,7 @@ func GetHubIdx(hub *csconfig.Hub) error { return fmt.Errorf("unable to read index file: %w", err) } - ret, err := LoadPkgIndex(bidx) + ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { return fmt.Errorf("unable to load existing index: %w", err) @@ -495,7 +487,7 @@ func GetHubIdx(hub *csconfig.Hub) error { return err } - hubIdx = ret + hubIdx = HubIndex{Items: ret} _, err = LocalSync(hub) if err != nil { @@ -504,52 +496,3 @@ func GetHubIdx(hub *csconfig.Hub) error { return nil } - -// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections -func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) { - var ( - RawIndex map[string]map[string]Item - missingItems []string - ) - - if err := json.Unmarshal(buff, &RawIndex); err != nil { - return nil, fmt.Errorf("failed to unmarshal index: %w", err) - } - - log.Debugf("%d item types in hub index", len(ItemTypes)) - - // Iterate over the different types to complete the struct - for _, itemType := range ItemTypes { - log.Tracef("%d item", len(RawIndex[itemType])) - - for name, item := range RawIndex[itemType] { - item.Name = name - item.Type = itemType - x := strings.Split(item.RemotePath, "/") - item.FileName = x[len(x)-1] - RawIndex[itemType][name] = item - - if itemType != COLLECTIONS { - continue - } - - // if it's a collection, check its sub-items are present - // XXX should be done later - for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections, item.WafRules} { - ptrtype := ItemTypes[idx] - for _, p := range ptr { - if _, ok := RawIndex[ptrtype][p]; !ok { - log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name) - missingItems = append(missingItems, p) - } - } - } - } - } - - if len(missingItems) > 0 { - return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference) - } - - return RawIndex, nil -} diff --git a/pkg/hubtest/coverage.go b/pkg/hubtest/coverage.go index eeff24b57..29db52715 100644 --- a/pkg/hubtest/coverage.go +++ b/pkg/hubtest/coverage.go @@ -27,12 +27,12 @@ type ScenarioCoverage struct { func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { var coverage []ParserCoverage - if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok { + if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok { return coverage, fmt.Errorf("no parsers in hub index") } //populate from hub, iterate in alphabetical order var pkeys []string - for pname := range h.HubIndex.Data[cwhub.PARSERS] { + for pname := range h.HubIndex.Items[cwhub.PARSERS] { pkeys = append(pkeys, pname) } sort.Strings(pkeys) @@ -100,12 +100,12 @@ func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) { var coverage []ScenarioCoverage - if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok { + if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok { return coverage, fmt.Errorf("no scenarios in hub index") } //populate from hub, iterate in alphabetical order var pkeys []string - for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] { + for scenarioName := range h.HubIndex.Items[cwhub.SCENARIOS] { pkeys = append(pkeys, scenarioName) } sort.Strings(pkeys) diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index c1aa4251c..eff2aa8fb 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -18,7 +18,7 @@ type HubTest struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *HubIndex + HubIndex *cwhub.HubIndex Tests []*HubTestItem } @@ -62,7 +62,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, } // load hub index - hubIndex, err := cwhub.LoadPkgIndex(bidx) + hubIndex, err := cwhub.ParseIndex(bidx) if err != nil { return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err) } @@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, TemplateConfigPath: templateConfigFilePath, TemplateProfilePath: templateProfilePath, TemplateSimulationPath: templateSimulationPath, - HubIndex: &HubIndex{Data: hubIndex}, + HubIndex: &cwhub.HubIndex{Items: hubIndex}, }, nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 47a151220..25a89d880 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -25,10 +25,6 @@ type HubTestItemConfig struct { OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } -type HubIndex struct { - Data map[string]map[string]cwhub.Item -} - type HubTestItem struct { Name string Path string @@ -43,7 +39,7 @@ type HubTestItem struct { RuntimeConfigFilePath string RuntimeProfileFilePath string RuntimeSimulationFilePath string - RuntimeHubConfig *csconfig.Hub + RuntimeHubConfig *csconfig.HubCfg ResultsPath string ParserResultFile string @@ -56,7 +52,7 @@ type HubTestItem struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *HubIndex + HubIndex *cwhub.HubIndex Config *HubTestItemConfig @@ -121,7 +117,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { ParserResultFile: filepath.Join(resultPath, ParserResultFileName), ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName), BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName), - RuntimeHubConfig: &csconfig.Hub{ + RuntimeHubConfig: &csconfig.HubCfg{ HubDir: runtimeHubFolder, HubIndexFile: hubTest.HubIndexFile, InstallDir: runtimeFolder, @@ -148,7 +144,7 @@ func (t *HubTestItem) InstallHub() error { continue } var parserDirDest string - if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok { + if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok { parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err) @@ -232,7 +228,7 @@ func (t *HubTestItem) InstallHub() error { continue } var scenarioDirDest string - if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok { + if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok { scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path to: %s", scenarioSource) @@ -301,7 +297,7 @@ func (t *HubTestItem) InstallHub() error { continue } var postoverflowDirDest string - if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok { + if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok { postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath)) if err != nil { return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err) @@ -423,7 +419,7 @@ func (t *HubTestItem) InstallHub() error { } // install data for postoverflows if needed - ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW) + ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { if item.Installed { if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 2e4a8035b..48b09795b 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -64,7 +64,7 @@ func NewParsers() *Parsers { StageFiles: make([]Stagefile, 0), PovfwStageFiles: make([]Stagefile, 0), } - for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} { + for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { for _, hubParserItem := range cwhub.GetItemMap(itemType) { if hubParserItem.Installed { stagefile := Stagefile{ @@ -74,7 +74,7 @@ func NewParsers() *Parsers { if itemType == cwhub.PARSERS { parsers.StageFiles = append(parsers.StageFiles, stagefile) } - if itemType == cwhub.PARSERS_OVFLW { + if itemType == cwhub.POSTOVERFLOWS { parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) } } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 92a1968c8..4b6034009 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -52,10 +52,6 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error return err } - if err := csConfig.LoadHub(); err != nil { - return fmt.Errorf("loading hub: %w", err) - } - cwhub.SetHubBranch() if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { @@ -121,7 +117,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { + if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) } } diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 2e38e0e6c..7bf27670b 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -55,16 +55,16 @@ teardown() { assert_stderr --partial "unable to create database client: unknown database type 'meh'" } -@test "crowdsec - bad configuration (empty/missing common section)" { +@test "crowdsec - default logging configuration (empty/missing common section)" { config_set '.common={}' - rune -1 "${CROWDSEC}" + rune -124 timeout 1s "${CROWDSEC}" refute_output - assert_stderr --partial "unable to load configuration: common section is empty" + assert_stderr --partial "Starting processing data" config_set 'del(.common)' - rune -1 "${CROWDSEC}" + rune -124 timeout 1s "${CROWDSEC}" refute_output - assert_stderr --partial "unable to load configuration: common section is empty" + assert_stderr --partial "Starting processing data" } @test "CS_LAPI_SECRET not strong enough" { diff --git a/test/bats/08_metrics.bats b/test/bats/08_metrics.bats index 836e22048..0275d7fd4 100644 --- a/test/bats/08_metrics.bats +++ b/test/bats/08_metrics.bats @@ -25,8 +25,7 @@ teardown() { @test "cscli metrics (crowdsec not running)" { rune -1 cscli metrics # crowdsec is down - assert_stderr --partial "failed to fetch prometheus metrics" - assert_stderr --partial "connect: connection refused" + assert_stderr --partial 'failed to fetch prometheus metrics: executing GET request for URL \"http://127.0.0.1:6060/metrics\" failed: Get \"http://127.0.0.1:6060/metrics\": dial tcp 127.0.0.1:6060: connect: connection refused' } @test "cscli metrics (bad configuration)" { @@ -43,18 +42,20 @@ teardown() { @test "cscli metrics (missing listen_addr)" { config_set 'del(.prometheus.listen_addr)' - rune -1 cscli metrics - assert_stderr --partial "no prometheus url, please specify" + rune -0 ./instance-crowdsec start + rune -0 cscli metrics --debug + assert_stderr --partial "prometheus.listen_addr is empty, defaulting to 127.0.0.1" } @test "cscli metrics (missing listen_port)" { - config_set 'del(.prometheus.listen_addr)' - rune -1 cscli metrics - assert_stderr --partial "no prometheus url, please specify" + config_set 'del(.prometheus.listen_port)' + rune -0 ./instance-crowdsec start + rune -0 cscli metrics --debug + assert_stderr --partial "prometheus.listen_port is empty or zero, defaulting to 6060" } @test "cscli metrics (missing prometheus section)" { config_set 'del(.prometheus)' rune -1 cscli metrics - assert_stderr --partial "prometheus section missing, can't show metrics" + assert_stderr --partial "prometheus is not enabled, can't show metrics" } diff --git a/test/bats/20_collections.bats b/test/bats/20_collections.bats deleted file mode 100644 index aa1fa6b21..000000000 --- a/test/bats/20_collections.bats +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bats -# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: - -set -u - -setup_file() { - load "../lib/setup_file.sh" -} - -teardown_file() { - load "../lib/teardown_file.sh" -} - -setup() { - load "../lib/setup.sh" - ./instance-data load - ./instance-crowdsec start -} - -teardown() { - ./instance-crowdsec stop -} - -#---------- - -@test "we can list collections" { - rune -0 cscli collections list -} - -@test "there are 2 collections (linux and sshd)" { - rune -0 cscli collections list -o json - rune -0 jq '.collections | length' <(output) - assert_output 2 -} - -@test "can install a collection (as a regular user) and remove it" { - # collection is not installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/mysql" - - # we install it - rune -0 cscli collections install crowdsecurity/mysql -o human - assert_stderr --partial "Enabled crowdsecurity/mysql" - - # it has been installed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - assert_line "crowdsecurity/mysql" - - # we install it - rune -0 cscli collections remove crowdsecurity/mysql -o human - assert_stderr --partial "Removed symlink [crowdsecurity/mysql]" - - # it has been removed - rune -0 cscli collections list -o json - rune -0 jq -r '.collections[].name' <(output) - refute_line "crowdsecurity/mysql" -} - -@test "must use --force to remove a collection that belongs to another, which becomes tainted" { - # we expect no error since we may have multiple collections, some removed and some not - rune -0 cscli collections remove crowdsecurity/sshd - assert_stderr --partial "crowdsecurity/sshd belongs to other collections" - assert_stderr --partial "[crowdsecurity/linux]" - - rune -0 cscli collections remove crowdsecurity/sshd --force - assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" - rune -0 cscli collections inspect crowdsecurity/linux -o json - rune -0 jq -r '.tainted' <(output) - assert_output "true" -} - -@test "can remove a collection" { - rune -0 cscli collections remove crowdsecurity/linux - assert_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." - rune -0 cscli collections inspect crowdsecurity/linux -o human - assert_line 'installed: false' -} - -@test "collections delete is an alias for collections remove" { - rune -0 cscli collections delete crowdsecurity/linux - assert_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." -} - -@test "removing a collection that does not exist is noop" { - rune -0 cscli collections remove crowdsecurity/apache2 - refute_stderr --partial "Removed" - assert_stderr --regexp ".*for the new configuration to be effective." -} - -@test "can remove a removed collection" { - rune -0 cscli collections install crowdsecurity/mysql - rune -0 cscli collections remove crowdsecurity/mysql - assert_stderr --partial "Removed" - rune -0 cscli collections remove crowdsecurity/mysql - refute_stderr --partial "Removed" -} - -@test "can remove all collections" { - # we may have this too, from package installs - rune cscli parsers delete crowdsecurity/whitelists - rune -0 cscli collections remove --all - assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" - assert_stderr --partial "Removed symlink [crowdsecurity/linux]" - rune -0 cscli hub list -o json - assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}' - rune -0 cscli collections remove --all - assert_stderr --partial 'Disabled 0 items' -} - -@test "a taint bubbles up to the top collection" { - coll=crowdsecurity/nginx - subcoll=crowdsecurity/base-http-scenarios - scenario=crowdsecurity/http-crawl-non_statics - - # install a collection with dependencies - rune -0 cscli collections install "$coll" - - # the collection, subcollection and scenario are installed and not tainted - # we have to default to false because tainted is (as of 1.4.6) returned - # only when true - rune -0 cscli collections inspect "$coll" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - rune -0 cscli collections inspect "$subcoll" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - rune -0 cscli scenarios inspect "$scenario" -o json - rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) - - # we taint the scenario - HUB_DIR=$(config_get '.config_paths.hub_dir') - yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml" - - # the collection, subcollection and scenario are now tainted - rune -0 cscli scenarios inspect "$scenario" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) - rune -0 cscli collections inspect "$subcoll" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) - rune -0 cscli collections inspect "$coll" -o json - rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) -} - -# TODO test download-only diff --git a/test/bats/20_hub_collections.bats b/test/bats/20_hub_collections.bats new file mode 100644 index 000000000..db33ae97f --- /dev/null +++ b/test/bats/20_hub_collections.bats @@ -0,0 +1,319 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli collections list" { + # no items + rune -0 cscli collections list + assert_output --partial "COLLECTIONS" + rune -0 cscli collections list -o json + assert_json '{collections:[]}' + rune -0 cscli collections list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + rune -0 cscli collections list + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli collections list -o json + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 jq '.collections | length' <(output) + assert_output "2" + + rune -0 cscli collections list -o raw + assert_output --partial crowdsecurity/sshd + assert_output --partial crowdsecurity/smb + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli collections list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.collections | length') + + rune -0 cscli collections list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli collections list -o json -a + rune -0 jq '.collections | length' <(output) + assert_output "$expected" + + rune -0 cscli collections list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + + +@test "cscli collections list [collection]..." { + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + # list one item + rune -0 cscli collections list crowdsecurity/sshd + assert_output --partial "crowdsecurity/sshd" + refute_output --partial "crowdsecurity/smb" + + # list multiple items + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb + assert_output --partial "crowdsecurity/sshd" + assert_output --partial "crowdsecurity/smb" + + rune -0 cscli collections list crowdsecurity/sshd -o json + rune -0 jq '.collections | length' <(output) + assert_output "1" + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o json + rune -0 jq '.collections | length' <(output) + assert_output "2" + + rune -0 cscli collections list crowdsecurity/sshd -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli collections list crowdsecurity/sshd crowdsecurity/smb -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli collections list [collection]... (not installed / not existing)" { + skip "not implemented yet" + # not installed + rune -1 cscli collections list crowdsecurity/sshd + # not existing + rune -1 cscli collections list blahblah/blahblah +} + +@test "cscli collections install [collection]..." { + rune -1 cscli collections install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli collections install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in collections" + + # simple install + rune -0 cscli collections install crowdsecurity/sshd + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli collections install crowdsecurity/ssshd + assert_stderr --partial "can't find 'crowdsecurity/ssshd' in collections, did you mean crowdsecurity/sshd?" + + # install multiple + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'installed: true' + rune -0 cscli collections inspect crowdsecurity/smb --no-metrics + assert_output --partial 'crowdsecurity/smb' + assert_output --partial 'installed: true' +} + +@test "cscli collections install [collection]... (file location and download-only)" { + # simple install + rune -0 cscli collections install crowdsecurity/linux --download-only + rune -0 cscli collections inspect crowdsecurity/linux --no-metrics + assert_output --partial 'crowdsecurity/linux' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/collections/crowdsecurity/linux.yaml" + assert_file_not_exists "$CONFIG_DIR/collections/linux.yaml" + + rune -0 cscli collections install crowdsecurity/linux + assert_file_exists "$CONFIG_DIR/collections/linux.yaml" +} + + +@test "cscli collections inspect [collection]..." { + rune -1 cscli collections inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli collections inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + # one item + rune -0 cscli collections inspect crowdsecurity/sshd --no-metrics + assert_line 'type: collections' + assert_line 'name: crowdsecurity/sshd' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: collections/crowdsecurity/sshd.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli collections inspect crowdsecurity/sshd + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null]' + + # one item, raw + rune -0 cscli collections inspect crowdsecurity/sshd -o raw + assert_line 'type: collections' + assert_line 'name: crowdsecurity/sshd' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: collections/crowdsecurity/sshd.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb --no-metrics + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'crowdsecurity/smb' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o json + rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) + assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",null]]' + + # multiple items, raw + rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw + assert_output --partial 'crowdsecurity/sshd' + assert_output --partial 'crowdsecurity/smb' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli collections remove [collection]..." { + rune -1 cscli collections remove + assert_stderr --partial "specify at least one collection to remove or '--all'" + + rune -1 cscli collections remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli collections remove crowdsecurity/sshd + + # install, then remove, check files + rune -0 cscli collections install crowdsecurity/sshd + assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections remove crowdsecurity/sshd + assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" + + # delete is an alias for remove + rune -0 cscli collections install crowdsecurity/sshd + assert_file_exists "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections delete crowdsecurity/sshd + assert_file_not_exists "$CONFIG_DIR/collections/sshd.yaml" + + # purge + assert_file_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" + rune -0 cscli collections remove crowdsecurity/sshd --purge + assert_file_not_exists "$HUB_DIR/collections/crowdsecurity/sshd.yaml" + + rune -0 cscli collections install crowdsecurity/sshd crowdsecurity/smb + + # --all + rune -0 cscli collections list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli collections remove --all + + rune -0 cscli collections list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli collections upgrade [collection]..." { + rune -1 cscli collections upgrade + assert_stderr --partial "specify at least one collection to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli collections upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in collections" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli collections upgrade crowdsecurity/sshd + assert_stderr --partial "can't find 'crowdsecurity/sshd' in collections" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/sshd":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli collections install crowdsecurity/sshd + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli collections upgrade crowdsecurity/sshd + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + # XXX: should return error + rune -0 cscli collections upgrade crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd is tainted, --force to overwrite" + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli collections upgrade crowdsecurity/sshd --force + rune -0 cscli collections inspect crowdsecurity/sshd -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli collections install crowdsecurity/smb + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + echo "dirty" >"$CONFIG_DIR/collections/smb.yaml" + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].local_version]==["?","?"]' <(output) + rune -0 cscli collections upgrade crowdsecurity/sshd crowdsecurity/smb + rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + echo "dirty" >"$CONFIG_DIR/collections/smb.yaml" + rune -0 cscli collections upgrade --all + rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output) +} diff --git a/test/bats/20_hub_collections_dep.bats b/test/bats/20_hub_collections_dep.bats new file mode 100644 index 000000000..26844c965 --- /dev/null +++ b/test/bats/20_hub_collections_dep.bats @@ -0,0 +1,86 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli collections (dependencies)" { + # inject a dependency: smb requires sshd + hub_dep=$(jq <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/smb":{collections:["crowdsecurity/sshd"]}}}') + echo "$hub_dep" >"$HUB_DIR/.index.json" + + # verify that installing smb brings sshd + rune -0 cscli collections install crowdsecurity/smb + rune -0 cscli collections list -o json + rune -0 jq -e '[.collections[].name]==["crowdsecurity/smb","crowdsecurity/sshd"]' <(output) + + # verify that removing smb removes sshd too + rune -0 cscli collections remove crowdsecurity/smb + rune -0 cscli collections list -o json + rune -0 jq -e '.collections | length == 0' <(output) + + # we can't remove sshd without --force + rune -0 cscli collections install crowdsecurity/smb + # XXX: should this be an error? + rune -0 cscli collections remove crowdsecurity/sshd + assert_stderr --partial "crowdsecurity/sshd belongs to other collections: [crowdsecurity/smb]" + assert_stderr --partial "Run 'sudo cscli collections remove crowdsecurity/sshd --force' if you want to force remove this sub collection" + rune -0 cscli collections list -o json + rune -0 jq -c '[.collections[].name]' <(output) + assert_json '["crowdsecurity/smb","crowdsecurity/sshd"]' + + # use the --force + rune -0 cscli collections remove crowdsecurity/sshd --force + rune -0 cscli collections list -o json + rune -0 jq -c '[.collections[].name]' <(output) + assert_json '["crowdsecurity/smb"]' + + # and now smb is tainted! + rune -0 cscli collections inspect crowdsecurity/smb -o json + rune -0 jq -e '.tainted//false==true' <(output) + rune -0 cscli collections remove crowdsecurity/smb --force + + # empty + rune -0 cscli collections list -o json + rune -0 jq -e '.collections | length == 0' <(output) + + # reinstall + rune -0 cscli collections install crowdsecurity/smb --force + + # taint on sshd means smb is tainted as well + rune -0 cscli collections inspect crowdsecurity/smb -o json + jq -e '.tainted//false==false' <(output) + echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml" + rune -0 cscli collections inspect crowdsecurity/smb -o json + jq -e '.tainted//false==true' <(output) + + # now we can't remove smb without --force + rune -1 cscli collections remove crowdsecurity/smb + assert_stderr --partial "unable to disable crowdsecurity/smb: crowdsecurity/smb is tainted, use '--force' to overwrite" +} diff --git a/test/bats/20_hub_parsers.bats b/test/bats/20_hub_parsers.bats new file mode 100644 index 000000000..77c3dc089 --- /dev/null +++ b/test/bats/20_hub_parsers.bats @@ -0,0 +1,417 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + # XXX: remove all "content" fields from the index, to make sure + # XXX: we don't rely on it in any way + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli parsers list" { + # no items + rune -0 cscli parsers list + assert_output --partial "PARSERS" + rune -0 cscli parsers list -o json + assert_json '{parsers:[]}' + rune -0 cscli parsers list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + rune -0 cscli parsers list + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli parsers list -o json + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 jq '.parsers | length' <(output) + assert_output "2" + + rune -0 cscli parsers list -o raw + assert_output --partial crowdsecurity/whitelists + assert_output --partial crowdsecurity/windows-auth + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli parsers list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.parsers | length') + + rune -0 cscli parsers list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o json -a + rune -0 jq '.parsers | length' <(output) + assert_output "$expected" + + rune -0 cscli parsers list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + +@test "cscli parsers list [parser]..." { + # non-existent + rune -1 cscli parsers install foo/bar + assert_stderr --partial "can't find 'foo/bar' in parsers" + + # not installed + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp 'crowdsecurity/whitelists.*disabled' + + # install two items + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + # list an installed item + rune -0 cscli parsers list crowdsecurity/whitelists + assert_output --regexp "crowdsecurity/whitelists.*enabled" + refute_output --partial "crowdsecurity/windows-auth" + + # list multiple installed and non installed items + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs + assert_output --partial "crowdsecurity/whitelists" + assert_output --partial "crowdsecurity/windows-auth" + assert_output --partial "crowdsecurity/traefik-logs" + + rune -0 cscli parsers list crowdsecurity/whitelists -o json + rune -0 jq '.parsers | length' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o json + rune -0 jq '.parsers | length' <(output) + assert_output "3" + + rune -0 cscli parsers list crowdsecurity/whitelists -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli parsers list crowdsecurity/whitelists crowdsecurity/windows-auth crowdsecurity/traefik-logs -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "3" +} + +@test "cscli parsers install [parser]..." { + rune -1 cscli parsers install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli parsers install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in parsers" + + # simple install + rune -0 cscli parsers install crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli parsers install crowdsecurity/sshd-logz + assert_stderr --partial "can't find 'crowdsecurity/sshd-logz' in parsers, did you mean crowdsecurity/sshd-logs?" + + # install multiple + rune -0 cscli parsers install crowdsecurity/pgsql-logs crowdsecurity/postfix-logs + rune -0 cscli parsers inspect crowdsecurity/pgsql-logs --no-metrics + assert_output --partial 'crowdsecurity/pgsql-logs' + assert_output --partial 'installed: true' + rune -0 cscli parsers inspect crowdsecurity/postfix-logs --no-metrics + assert_output --partial 'crowdsecurity/postfix-logs' + assert_output --partial 'installed: true' +} + +@test "cscli parsers install [parser]... (file location and download-only)" { + # simple install + rune -0 cscli parsers install crowdsecurity/whitelists --download-only + rune -0 cscli parsers inspect crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/whitelists' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" +} + +# XXX: test install with --force +# XXX: test install with --ignore + +@test "cscli parsers inspect [parser]..." { + rune -1 cscli parsers inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli parsers inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # one item + rune -0 cscli parsers inspect crowdsecurity/sshd-logs --no-metrics + assert_line 'type: parsers' + assert_line 'stage: s01-parse' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json + rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null]' + + # one item, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw + assert_line 'type: parsers' + assert_line 'stage: s01-parse' + assert_line 'name: crowdsecurity/sshd-logs' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists --no-metrics + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json + rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) + assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",null]]' + + # multiple items, raw + rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw + assert_output --partial 'crowdsecurity/sshd-logs' + assert_output --partial 'crowdsecurity/whitelists' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli parsers remove [parser]..." { + rune -1 cscli parsers remove + assert_stderr --partial "specify at least one parser to remove or '--all'" + + rune -1 cscli parsers remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli parsers remove crowdsecurity/whitelists + + # XXX: have the --force ignore uninstalled items + # XXX: maybe also with --purge + + # install, then remove, check files + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers remove crowdsecurity/whitelists + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + # delete is an alias for remove + rune -0 cscli parsers install crowdsecurity/whitelists + assert_file_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers delete crowdsecurity/whitelists + assert_file_not_exists "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + + # purge + assert_file_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + rune -0 cscli parsers remove crowdsecurity/whitelists --purge + assert_file_not_exists "$HUB_DIR/parsers/s02-enrich/crowdsecurity/whitelists.yaml" + + rune -0 cscli parsers install crowdsecurity/whitelists crowdsecurity/windows-auth + + # --all + rune -0 cscli parsers list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli parsers remove --all + + rune -0 cscli parsers list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli parsers upgrade [parser]..." { + rune -1 cscli parsers upgrade + assert_stderr --partial "specify at least one parser to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli parsers upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in parsers" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_stderr --partial "can't find 'crowdsecurity/whitelists' in parsers" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {parsers:{"crowdsecurity/whitelists":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli parsers install crowdsecurity/whitelists + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli parsers upgrade crowdsecurity/whitelists + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + # XXX: should return error + rune -0 cscli parsers upgrade crowdsecurity/whitelists + assert_stderr --partial "crowdsecurity/whitelists is tainted, --force to overwrite" + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli parsers upgrade crowdsecurity/whitelists --force + rune -0 cscli parsers inspect crowdsecurity/whitelists -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli parsers install crowdsecurity/windows-auth + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + rune -0 cscli parsers list -o json + rune -0 jq -e '[.parsers[].local_version]==["?","?"]' <(output) + rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/windows-auth + rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml" + echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml" + rune -0 cscli parsers upgrade --all + rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output) +} + + + +#@test "must use --force to remove a collection that belongs to another, which becomes tainted" { +# # we expect no error since we may have multiple collections, some removed and some not +# rune -0 cscli collections remove crowdsecurity/sshd +# assert_stderr --partial "crowdsecurity/sshd belongs to other collections" +# assert_stderr --partial "[crowdsecurity/linux]" +# +# rune -0 cscli collections remove crowdsecurity/sshd --force +# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" +# rune -0 cscli collections inspect crowdsecurity/linux -o json +# rune -0 jq -r '.tainted' <(output) +# assert_output "true" +#} +# +#@test "can remove a collection" { +# rune -0 cscli collections remove crowdsecurity/linux +# assert_stderr --partial "Removed" +# assert_stderr --regexp ".*for the new configuration to be effective." +# rune -0 cscli collections inspect crowdsecurity/linux -o human --no-metrics +# assert_line 'installed: false' +#} +# +#@test "collections delete is an alias for collections remove" { +# rune -0 cscli collections delete crowdsecurity/linux +# assert_stderr --partial "Removed" +# assert_stderr --regexp ".*for the new configuration to be effective." +#} +# +#@test "removing a collection that does not exist is noop" { +# rune -0 cscli collections remove crowdsecurity/apache2 +# refute_stderr --partial "Removed" +# assert_stderr --regexp ".*for the new configuration to be effective." +#} +# +#@test "can remove a removed collection" { +# rune -0 cscli collections install crowdsecurity/mysql +# rune -0 cscli collections remove crowdsecurity/mysql +# assert_stderr --partial "Removed" +# rune -0 cscli collections remove crowdsecurity/mysql +# refute_stderr --partial "Removed" +#} +# +#@test "can remove all collections" { +# # we may have this too, from package installs +# rune cscli parsers delete crowdsecurity/whitelists +# rune -0 cscli collections remove --all +# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]" +# assert_stderr --partial "Removed symlink [crowdsecurity/linux]" +# rune -0 cscli hub list -o json +# assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}' +# rune -0 cscli collections remove --all +# assert_stderr --partial 'Disabled 0 items' +#} +# +#@test "a taint bubbles up to the top collection" { +# coll=crowdsecurity/nginx +# subcoll=crowdsecurity/base-http-scenarios +# scenario=crowdsecurity/http-crawl-non_statics +# +# # install a collection with dependencies +# rune -0 cscli collections install "$coll" +# +# # the collection, subcollection and scenario are installed and not tainted +# # we have to default to false because tainted is (as of 1.4.6) returned +# # only when true +# rune -0 cscli collections inspect "$coll" -o json +# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) +# rune -0 cscli collections inspect "$subcoll" -o json +# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) +# rune -0 cscli scenarios inspect "$scenario" -o json +# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output) +# +# # we taint the scenario +# HUB_DIR=$(config_get '.config_paths.hub_dir') +# yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml" +# +# # the collection, subcollection and scenario are now tainted +# rune -0 cscli scenarios inspect "$scenario" -o json +# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) +# rune -0 cscli collections inspect "$subcoll" -o json +# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) +# rune -0 cscli collections inspect "$coll" -o json +# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output) +#} +# +## TODO test download-only diff --git a/test/bats/20_hub_postoverflows.bats b/test/bats/20_hub_postoverflows.bats new file mode 100644 index 000000000..cf74268f7 --- /dev/null +++ b/test/bats/20_hub_postoverflows.bats @@ -0,0 +1,321 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli postoverflows list" { + # no items + rune -0 cscli postoverflows list + assert_output --partial "POSTOVERFLOWS" + rune -0 cscli postoverflows list -o json + assert_json '{postoverflows:[]}' + rune -0 cscli postoverflows list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + rune -0 cscli postoverflows list + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli postoverflows list -o json + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 jq '.postoverflows | length' <(output) + assert_output "2" + + rune -0 cscli postoverflows list -o raw + assert_output --partial crowdsecurity/rdns + assert_output --partial crowdsecurity/cdn-whitelist + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli postoverflows list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.postoverflows | length') + + rune -0 cscli postoverflows list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli postoverflows list -o json -a + rune -0 jq '.postoverflows | length' <(output) + assert_output "$expected" + + rune -0 cscli postoverflows list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + + +@test "cscli postoverflows list [scenario]..." { + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + # list one item + rune -0 cscli postoverflows list crowdsecurity/rdns + assert_output --partial "crowdsecurity/rdns" + refute_output --partial "crowdsecurity/cdn-whitelist" + + # list multiple items + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist + assert_output --partial "crowdsecurity/rdns" + assert_output --partial "crowdsecurity/cdn-whitelist" + + rune -0 cscli postoverflows list crowdsecurity/rdns -o json + rune -0 jq '.postoverflows | length' <(output) + assert_output "1" + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json + rune -0 jq '.postoverflows | length' <(output) + assert_output "2" + + rune -0 cscli postoverflows list crowdsecurity/rdns -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli postoverflows list crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli postoverflows list [scenario]... (not installed / not existing)" { + skip "not implemented yet" + # not installed + rune -1 cscli postoverflows list crowdsecurity/rdns + # not existing + rune -1 cscli postoverflows list blahblah/blahblah +} + +@test "cscli postoverflows install [scenario]..." { + rune -1 cscli postoverflows install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli postoverflows install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in postoverflows" + + # simple install + rune -0 cscli postoverflows install crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli postoverflows install crowdsecurity/rdnf + assert_stderr --partial "can't find 'crowdsecurity/rdnf' in postoverflows, did you mean crowdsecurity/rdns?" + + # install multiple + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: true' + rune -0 cscli postoverflows inspect crowdsecurity/cdn-whitelist --no-metrics + assert_output --partial 'crowdsecurity/cdn-whitelist' + assert_output --partial 'installed: true' +} + +@test "cscli postoverflows install [postoverflow]... (file location and download-only)" { + # simple install + rune -0 cscli postoverflows install crowdsecurity/rdns --download-only + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" +} + + +@test "cscli postoverflows inspect [scenario]..." { + rune -1 cscli postoverflows inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli postoverflows inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + # one item + rune -0 cscli postoverflows inspect crowdsecurity/rdns --no-metrics + assert_line 'type: postoverflows' + assert_line 'stage: s00-enrich' + assert_line 'name: crowdsecurity/rdns' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli postoverflows inspect crowdsecurity/rdns + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null]' + + # one item, raw + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw + assert_line 'type: postoverflows' + assert_line 'stage: s00-enrich' + assert_line 'name: crowdsecurity/rdns' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist --no-metrics + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'crowdsecurity/cdn-whitelist' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json + rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output) + assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",null]]' + + # multiple items, raw + rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw + assert_output --partial 'crowdsecurity/rdns' + assert_output --partial 'crowdsecurity/cdn-whitelist' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli postoverflows remove [postoverflow]..." { + rune -1 cscli postoverflows remove + assert_stderr --partial "specify at least one postoverflow to remove or '--all'" + + rune -1 cscli postoverflows remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli postoverflows remove crowdsecurity/rdns + + # install, then remove, check files + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows remove crowdsecurity/rdns + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + # delete is an alias for remove + rune -0 cscli postoverflows install crowdsecurity/rdns + assert_file_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows delete crowdsecurity/rdns + assert_file_not_exists "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + + # purge + assert_file_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + rune -0 cscli postoverflows remove crowdsecurity/rdns --purge + assert_file_not_exists "$HUB_DIR/postoverflows/s00-enrich/crowdsecurity/rdns.yaml" + + rune -0 cscli postoverflows install crowdsecurity/rdns crowdsecurity/cdn-whitelist + + # --all + rune -0 cscli postoverflows list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli postoverflows remove --all + + rune -0 cscli postoverflows list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli postoverflows upgrade [postoverflow]..." { + rune -1 cscli postoverflows upgrade + assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli postoverflows upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + assert_stderr --partial "can't find 'crowdsecurity/rdns' in postoverflows" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {postoverflows:{"crowdsecurity/rdns":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli postoverflows install crowdsecurity/rdns + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + # XXX: should return error + rune -0 cscli postoverflows upgrade crowdsecurity/rdns + assert_stderr --partial "crowdsecurity/rdns is tainted, --force to overwrite" + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli postoverflows upgrade crowdsecurity/rdns --force + rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli postoverflows install crowdsecurity/cdn-whitelist + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + rune -0 cscli postoverflows list -o json + rune -0 jq -e '[.postoverflows[].local_version]==["?","?"]' <(output) + rune -0 cscli postoverflows upgrade crowdsecurity/rdns crowdsecurity/cdn-whitelist + rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml" + echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml" + rune -0 cscli postoverflows upgrade --all + rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output) +} diff --git a/test/bats/20_hub_scenarios.bats b/test/bats/20_hub_scenarios.bats new file mode 100644 index 000000000..2eeb6146b --- /dev/null +++ b/test/bats/20_hub_scenarios.bats @@ -0,0 +1,321 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + CONFIG_DIR=$(config_get '.config_paths.config_dir') + export CONFIG_DIR +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + ./instance-data load + hub_uninstall_all + hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)') + echo "$hub_min" >"$HUB_DIR/.index.json" +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli scenarios list" { + # no items + rune -0 cscli scenarios list + assert_output --partial "SCENARIOS" + rune -0 cscli scenarios list -o json + assert_json '{scenarios:[]}' + rune -0 cscli scenarios list -o raw + assert_output 'name,status,version,description' + + # some items + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + rune -0 cscli scenarios list + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 grep -c enabled <(output) + assert_output "2" + + rune -0 cscli scenarios list -o json + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 jq '.scenarios | length' <(output) + assert_output "2" + + rune -0 cscli scenarios list -o raw + assert_output --partial crowdsecurity/ssh-bf + assert_output --partial crowdsecurity/telnet-bf + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" +} + +@test "cscli scenarios list -a" { + expected=$(jq <"$HUB_DIR/.index.json" -r '.scenarios | length') + + rune -0 cscli scenarios list -a + rune -0 grep -c disabled <(output) + assert_output "$expected" + + rune -0 cscli scenarios list -o json -a + rune -0 jq '.scenarios | length' <(output) + assert_output "$expected" + + rune -0 cscli scenarios list -o raw -a + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "$expected" +} + +@test "cscli scenarios list [scenario]..." { + # non-existent + rune -1 cscli scenario install foo/bar + assert_stderr --partial "can't find 'foo/bar' in scenarios" + + # not installed + rune -0 cscli scenarios list crowdsecurity/ssh-bf + assert_output --regexp 'crowdsecurity/ssh-bf.*disabled' + + # install two items + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + # list an installed item + rune -0 cscli scenarios list crowdsecurity/ssh-bf + assert_output --regexp "crowdsecurity/ssh-bf.*enabled" + refute_output --partial "crowdsecurity/telnet-bf" + + # list multiple installed and non installed items + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf crowdsecurity/aws-bf + assert_output --partial "crowdsecurity/ssh-bf" + assert_output --partial "crowdsecurity/telnet-bf" + assert_output --partial "crowdsecurity/aws-bf" + + rune -0 cscli scenarios list crowdsecurity/ssh-bf -o json + rune -0 jq '.scenarios | length' <(output) + assert_output "1" + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o json + rune -0 jq '.scenarios | length' <(output) + assert_output "3" + + rune -0 cscli scenarios list crowdsecurity/ssh-bf -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "1" + rune -0 cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/telnet-bf crowdsecurity/aws-bf -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "3" +} + +@test "cscli scenarios install [scenario]..." { + rune -1 cscli scenarios install + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + + # not in hub + rune -1 cscli scenarios install crowdsecurity/blahblah + assert_stderr --partial "can't find 'crowdsecurity/blahblah' in scenarios" + + # simple install + rune -0 cscli scenarios install crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: true' + + # autocorrect + rune -1 cscli scenarios install crowdsecurity/ssh-tf + assert_stderr --partial "can't find 'crowdsecurity/ssh-tf' in scenarios, did you mean crowdsecurity/ssh-bf?" + + # install multiple + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: true' + rune -0 cscli scenarios inspect crowdsecurity/telnet-bf --no-metrics + assert_output --partial 'crowdsecurity/telnet-bf' + assert_output --partial 'installed: true' +} + + +@test "cscli scenarios install [scenario]... (file location and download-only)" { + # simple install + rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'installed: false' + assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" +} + + +@test "cscli scenarios inspect [scenario]..." { + rune -1 cscli scenarios inspect + assert_stderr --partial 'requires at least 1 arg(s), only received 0' + ./instance-crowdsec start + + rune -1 cscli scenarios inspect blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + # one item + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf --no-metrics + assert_line 'type: scenarios' + assert_line 'name: crowdsecurity/ssh-bf' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # one item, with metrics + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf + assert_line --partial 'Current metrics:' + + # one item, json + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output) + # XXX: .installed is missing -- not false + assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null]' + + # one item, raw + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o raw + assert_line 'type: scenarios' + assert_line 'name: crowdsecurity/ssh-bf' + assert_line 'author: crowdsecurity' + assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml' + assert_line 'installed: false' + refute_line --partial 'Current metrics:' + + # multiple items + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf --no-metrics + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'crowdsecurity/telnet-bf' + rune -1 grep -c 'Current metrics:' <(output) + assert_output "0" + + # multiple items, with metrics + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 grep -c 'Current metrics:' <(output) + assert_output "2" + + # multiple items, json + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json + rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output) + assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",null]]' + + # multiple items, raw + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw + assert_output --partial 'crowdsecurity/ssh-bf' + assert_output --partial 'crowdsecurity/telnet-bf' + run -1 grep -c 'Current metrics:' <(output) + assert_output "0" +} + +@test "cscli scenarios remove [scenario]..." { + rune -1 cscli scenarios remove + assert_stderr --partial "specify at least one scenario to remove or '--all'" + + rune -1 cscli scenarios remove blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + # XXX: we can however remove a real item if it's not installed, or already removed + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + + # install, then remove, check files + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + # delete is an alias for remove + rune -0 cscli scenarios install crowdsecurity/ssh-bf + assert_file_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios delete crowdsecurity/ssh-bf + assert_file_not_exists "$CONFIG_DIR/scenarios/ssh-bf.yaml" + + # purge + assert_file_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge + assert_file_not_exists "$HUB_DIR/scenarios/crowdsecurity/ssh-bf.yaml" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/telnet-bf + + # --all + rune -0 cscli scenarios list -o raw + rune -0 grep -vc 'name,status,version,description' <(output) + assert_output "2" + + rune -0 cscli scenarios remove --all + + rune -0 cscli scenarios list -o raw + rune -1 grep -vc 'name,status,version,description' <(output) + assert_output "0" +} + +@test "cscli scenarios upgrade [scenario]..." { + rune -1 cscli scenarios upgrade + assert_stderr --partial "specify at least one scenario to upgrade or '--all'" + + # XXX: should this return 1 instead of log.Error? + rune -0 cscli scenarios upgrade blahblah/blahblah + assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios" + + # XXX: same message if the item exists but is not installed, this is confusing + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + assert_stderr --partial "can't find 'crowdsecurity/ssh-bf' in scenarios" + + # hash of an empty file + sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + # add version 0.0 to the hub + new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {scenarios:{"crowdsecurity/ssh-bf":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}') + echo "$new_hub" >"$HUB_DIR/.index.json" + + rune -0 cscli scenarios install crowdsecurity/ssh-bf + + # bring the file to v0.0 + truncate -s 0 "$CONFIG_DIR/scenarios/ssh-bf.yaml" + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version=="0.0"' <(output) + + # upgrade + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version==.version' <(output) + + # taint + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + # XXX: should return error + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf + assert_stderr --partial "crowdsecurity/ssh-bf is tainted, --force to overwrite" + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version=="?"' <(output) + + # force upgrade with taint + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf --force + rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json + rune -0 jq -e '.local_version==.version' <(output) + + # multiple items + rune -0 cscli scenarios install crowdsecurity/telnet-bf + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + rune -0 cscli scenarios list -o json + rune -0 jq -e '[.scenarios[].local_version]==["?","?"]' <(output) + rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/telnet-bf + rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output) + + # upgrade all + echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml" + echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml" + rune -0 cscli scenarios upgrade --all + rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output) +} diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 5e16340ec..bcd216091 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -238,6 +238,12 @@ assert_stderr_line() { } export -f assert_stderr_line +hub_uninstall_all() { + CONFIG_DIR=$(dirname "$CONFIG_YAML") + rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/* +} +export -f hub_uninstall_all + # remove color and style sequences from stdin plaintext() { sed -E 's/\x1B\[[0-9;]*[JKmsu]//g'