beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor hub management and cscli commands (#2545)

Browse source
This commit is contained in:
mmetc 2023-11-24 15:57:32 +01:00 committed by GitHub
parent 32e9eb4be4
commit ffcab0b2bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
124 changed files with 6836 additions and 4414 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmd/crowdsec-cli/capi.go
View file

@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command {
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err) return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
} }
if err := require.Hub(csConfig); err != nil { hub, err := require.Hub(csConfig, nil)
if err != nil {
return err return err
} }
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil { if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err) return fmt.Errorf("failed to get scenarios: %w", err)
} }

176
cmd/crowdsec-cli/collections.go
View file

@ -1,176 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCollectionsCmd() *cobra.Command {
var cmdCollections = &cobra.Command{
Use: "collections [action]",
Short: "Manage collections from hub",
Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
/*TBD fix help*/
Args: cobra.MinimumNArgs(1),
Aliases: []string{"collection"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
var ignoreError bool
var cmdCollectionsInstall = &cobra.Command{
Use: "install collection",
Short: "Install given collection(s)",
Long: `Fetch and install given collection(s) from hub`,
Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.COLLECTIONS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
cmdCollections.AddCommand(cmdCollectionsInstall)
var cmdCollectionsRemove = &cobra.Command{
Use: "remove collection",
Short: "Remove given collection(s)",
Long: `Remove given collection(s) from hub`,
Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one collection to remove or '--all'")
}
for _, name := range args {
if !forceAction {
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
if item == nil {
return fmt.Errorf("unable to retrieve: %s", name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
continue
}
}
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
}
return nil
},
}
cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
cmdCollections.AddCommand(cmdCollectionsRemove)
var cmdCollectionsUpgrade = &cobra.Command{
Use: "upgrade collection",
Short: "Upgrade given collection(s)",
Long: `Fetch and upgrade given collection(s) from hub`,
Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one collection to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
}
}
return nil
},
}
cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
cmdCollections.AddCommand(cmdCollectionsUpgrade)
var cmdCollectionsInspect = &cobra.Command{
Use: "inspect collection",
Short: "Inspect given collection",
Long: `Inspect given collection`,
Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
InspectItem(name, cwhub.COLLECTIONS)
}
},
}
cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
cmdCollections.AddCommand(cmdCollectionsInspect)
var cmdCollectionsList = &cobra.Command{
Use: "list collection [-a]",
Short: "List all collections",
Long: `List all collections`,
Example: `cscli collections list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
},
}
cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
cmdCollections.AddCommand(cmdCollectionsList)
return cmdCollections
}

38
cmd/crowdsec-cli/config_backup.go
View file

@ -14,21 +14,25 @@ import (
) )
func backupHub(dirPath string) error { func backupHub(dirPath string) error {
var err error
var itemDirectory string var itemDirectory string
var upstreamParsers []string var upstreamParsers []string
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes { for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{ clog := log.WithFields(log.Fields{
"type": itemType, "type": itemType,
}) })
itemMap := cwhub.GetItemMap(itemType) itemMap := hub.GetItemMap(itemType)
if itemMap == nil { if itemMap == nil {
clog.Infof("No %s to backup.", itemType) clog.Infof("No %s to backup.", itemType)
continue continue
} }
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType) itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil { if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err) return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
} }
upstreamParsers = []string{} upstreamParsers = []string{}
@ -36,30 +40,30 @@ func backupHub(dirPath string) error {
clog = clog.WithFields(log.Fields{ clog = clog.WithFields(log.Fields{
"file": v.Name, "file": v.Name,
}) })
if !v.Installed { //only backup installed ones if !v.State.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k) clog.Debugf("[%s] : not installed", k)
continue continue
} }
//for the local/tainted ones, we back up the full file //for the local/tainted ones, we back up the full file
if v.Tainted || v.Local || !v.UpToDate { if v.State.Tainted || v.IsLocal() || !v.State.UpToDate {
//we need to back up stages for parsers //we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW { if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage) fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil { if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err) return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
} }
} }
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil { if err = CopyFile(v.State.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
} }
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
continue continue
} }
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate) clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate) clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name) upstreamParsers = append(upstreamParsers, v.Name)
} }
//write the upstream items //write the upstream items
@ -100,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error {
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath) parentDir := filepath.Dir(dirPath)
if _, err := os.Stat(parentDir); err != nil { if _, err = os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
} }
@ -197,10 +201,6 @@ func backupConfigToDirectory(dirPath string) error {
} }
func runConfigBackup(cmd *cobra.Command, args []string) error { func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
if err := backupConfigToDirectory(args[0]); err != nil { if err := backupConfigToDirectory(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err) return fmt.Errorf("failed to backup config: %w", err)
} }

54
cmd/crowdsec-cli/config_restore.go
View file

@ -21,45 +21,12 @@ type OldAPICfg struct {
Password string `json:"password"` Password string `json:"password"`
} }
// it's a rip of the cli version, but in silent-mode
func silentInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
if downloadOnly && item.Downloaded && item.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
}
err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
}
err = cwhub.EnableItem(csConfig.Hub, item)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", item.Name), nil
}
func restoreHub(dirPath string) error { func restoreHub(dirPath string) error {
var err error hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
if err := csConfig.LoadHub(); err != nil {
return err return err
} }
cwhub.SetHubBranch()
for _, itype := range cwhub.ItemTypes { for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil { if _, err = os.Stat(itemDirectory); err != nil {
@ -78,13 +45,14 @@ func restoreHub(dirPath string) error {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
} }
for _, toinstall := range upstreamList { for _, toinstall := range upstreamList {
label, err := silentInstallItem(toinstall, itype) item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
err := item.Install(false, false)
if err != nil { if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err) log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
} }
} }
@ -98,7 +66,7 @@ func restoreHub(dirPath string) error {
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) { if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue continue
} }
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW { if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
//we expect a stage here //we expect a stage here
if !file.IsDir() { if !file.IsDir() {
continue continue
@ -302,10 +270,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return err return err
} }
if err := require.Hub(csConfig); err != nil {
return err
}
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil { if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", args[0], err) return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
} }

1
cmd/crowdsec-cli/config_show.go
View file

@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
cscli: cscli:
- Output : {{.Cscli.Output}} - Output : {{.Cscli.Output}}
- Hub Branch : {{.Cscli.HubBranch}} - Hub Branch : {{.Cscli.HubBranch}}
- Hub Folder : {{.Cscli.HubDir}}
{{- end }} {{- end }}
{{- if .API }} {{- if .API }}

5
cmd/crowdsec-cli/console.go
View file

@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba
return fmt.Errorf("could not parse CAPI URL: %s", err) return fmt.Errorf("could not parse CAPI URL: %s", err)
} }
if err := require.Hub(csConfig); err != nil { hub, err := require.Hub(csConfig, nil)
if err != nil {
return err return err
} }
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil { if err != nil {
return fmt.Errorf("failed to get installed scenarios: %s", err) return fmt.Errorf("failed to get installed scenarios: %s", err)
} }

208
cmd/crowdsec-cli/hub.go
View file

@ -1,7 +1,6 @@
package main package main
import ( import (
"errors"
"fmt" "fmt"
"github.com/fatih/color" "github.com/fatih/color"
@ -13,31 +12,20 @@ import (
) )
func NewHubCmd() *cobra.Command { func NewHubCmd() *cobra.Command {
var cmdHub = &cobra.Command{ cmdHub := &cobra.Command{
Use: "hub [action]", Use: "hub [action]",
Short: "Manage Hub", Short: "Manage hub index",
Long: ` Long: `Hub management
Hub management
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net). List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update. The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
`, Example: `cscli hub list
Example: ` cscli hub update
cscli hub list # List all installed configurations cscli hub upgrade`,
cscli hub update # Download list of available configurations from the hub
`,
Args: cobra.ExactArgs(0), Args: cobra.ExactArgs(0),
DisableAutoGenTag: true, DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
} }
return nil
},
}
cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
cmdHub.AddCommand(NewHubListCmd()) cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd()) cmdHub.AddCommand(NewHubUpdateCmd())
cmdHub.AddCommand(NewHubUpgradeCmd()) cmdHub.AddCommand(NewHubUpgradeCmd())
@ -45,116 +33,142 @@ cscli hub update # Download list of available configurations from the hub
return cmdHub return cmdHub
} }
func NewHubListCmd() *cobra.Command { func runHubList(cmd *cobra.Command, args []string) error {
var cmdHubList = &cobra.Command{ flags := cmd.Flags()
Use: "list [-a]",
Short: "List installed configs", all, err := flags.GetBool("all")
Args: cobra.ExactArgs(0), if err != nil {
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err return err
} }
// use LocalSync to get warnings about tainted / outdated items hub, err := require.Hub(csConfig, nil)
warn, _ := cwhub.LocalSync(csConfig.Hub) if err != nil {
for _, v := range warn { return err
}
for _, v := range hub.Warnings {
log.Info(v) log.Info(v)
} }
cwhub.DisplaySummary()
ListItems(color.Output, []string{ for _, line := range hub.ItemStats() {
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, log.Info(line)
}, args, true, false, all) }
items := make(map[string][]*cwhub.Item)
for _, itemType := range cwhub.ItemTypes {
items[itemType], err = selectItems(hub, itemType, nil, !all)
if err != nil {
return err
}
}
err = listItems(color.Output, cwhub.ItemTypes, items)
if err != nil {
return err
}
return nil return nil
}, }
func NewHubListCmd() *cobra.Command {
cmdHubList := &cobra.Command{
Use: "list [-a]",
Short: "List all installed configurations",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubList,
} }
cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
flags := cmdHubList.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
return cmdHubList return cmdHubList
} }
func NewHubUpdateCmd() *cobra.Command { func runHubUpdate(cmd *cobra.Command, args []string) error {
var cmdHubUpdate = &cobra.Command{ local := csConfig.Hub
Use: "update", remote := require.RemoteHub(csConfig)
Short: "Fetch available configs from hub",
Long: ` // don't use require.Hub because if there is no index file, it would fail
Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs. hub, err := cwhub.NewHub(local, remote, true)
`, if err != nil {
Args: cobra.ExactArgs(0), return fmt.Errorf("failed to update hub: %w", err)
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
} }
cwhub.SetHubBranch() for _, v := range hub.Warnings {
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)
}
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("failed to get Hub index after retry: %w", err)
}
}
// use LocalSync to get warnings about tainted / outdated items
warn, _ := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v) log.Info(v)
} }
return nil return nil
}, }
func NewHubUpdateCmd() *cobra.Command {
cmdHubUpdate := &cobra.Command{
Use: "update",
Short: "Download the latest index (catalog of available configurations)",
Long: `
Fetches the .index.json file from the hub, containing the list of available configs.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubUpdate,
} }
return cmdHubUpdate return cmdHubUpdate
} }
func runHubUpgrade(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
items, err := hub.GetInstalledItems(itemType)
if err != nil {
return err
}
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}
return nil
}
func NewHubUpgradeCmd() *cobra.Command { func NewHubUpgradeCmd() *cobra.Command {
var cmdHubUpgrade = &cobra.Command{ cmdHubUpgrade := &cobra.Command{
Use: "upgrade", Use: "upgrade",
Short: "Upgrade all configs installed from hub", Short: "Upgrade all configurations to their latest version",
Long: ` Long: `
Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available. Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
`, `,
Args: cobra.ExactArgs(0), Args: cobra.ExactArgs(0),
DisableAutoGenTag: true, DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error { RunE: runHubUpgrade,
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
} }
cwhub.SetHubBranch() flags := cmdHubUpgrade.Flags()
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
log.Infof("Upgrading collections")
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
log.Infof("Upgrading parsers")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
log.Infof("Upgrading scenarios")
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
log.Infof("Upgrading postoverflows")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
return nil
},
}
cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdHubUpgrade return cmdHubUpgrade
} }

66
cmd/crowdsec-cli/hubtest.go
View file

@ -18,9 +18,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/hubtest" "github.com/crowdsecurity/crowdsec/pkg/hubtest"
) )
var ( var HubTest hubtest.HubTest
HubTest hubtest.HubTest
)
func NewHubTestCmd() *cobra.Command { func NewHubTestCmd() *cobra.Command {
var hubPath string var hubPath string
@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command {
return nil return nil
}, },
} }
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder") cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec") cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli") cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command {
return cmdHubTest return cmdHubTest
} }
func NewHubTestCreateCmd() *cobra.Command { func NewHubTestCreateCmd() *cobra.Command {
parsers := []string{} parsers := []string{}
postoverflows := []string{} postoverflows := []string{}
@ -138,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
} }
configFilePath := filepath.Join(testPath, "config.yaml") configFilePath := filepath.Join(testPath, "config.yaml")
fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) fd, err := os.Create(configFilePath)
if err != nil { if err != nil {
return fmt.Errorf("open: %s", err) return fmt.Errorf("open: %s", err)
} }
@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return nil return nil
}, },
} }
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test") cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test") cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test") cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return cmdHubTestCreate return cmdHubTestCreate
} }
func NewHubTestRunCmd() *cobra.Command { func NewHubTestRunCmd() *cobra.Command {
var noClean bool var noClean bool
var runAll bool var runAll bool
@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
if !runAll && len(args) == 0 { if !runAll && len(args) == 0 {
printHelp(cmd) printHelp(cmd)
return fmt.Errorf("Please provide test to run or --all flag") return fmt.Errorf("please provide test to run or --all flag")
} }
if runAll { if runAll {
@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command {
} }
} }
// set timezone to avoid DST issues
os.Setenv("TZ", "UTC")
for _, test := range HubTest.Tests { for _, test := range HubTest.Tests {
if csConfig.Cscli.Output == "human" { if csConfig.Cscli.Output == "human" {
log.Infof("Running test '%s'", test.Name) log.Infof("Running test '%s'", test.Name)
@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command {
} }
} }
} }
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
hubTestResultTable(color.Output, testResult) hubTestResultTable(color.Output, testResult)
} else if csConfig.Cscli.Output == "json" { case "json":
jsonResult := make(map[string][]string, 0) jsonResult := make(map[string][]string, 0)
jsonResult["success"] = make([]string, 0) jsonResult["success"] = make([]string, 0)
jsonResult["fail"] = make([]string, 0) jsonResult["fail"] = make([]string, 0)
@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command {
return fmt.Errorf("unable to json test result: %s", err) return fmt.Errorf("unable to json test result: %s", err)
} }
fmt.Println(string(jsonStr)) fmt.Println(string(jsonStr))
default:
return fmt.Errorf("only human/json output modes are supported")
} }
if !success { if !success {
@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command {
return nil return nil
}, },
} }
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed") cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail") cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests") cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command {
return cmdHubTestRun return cmdHubTestRun
} }
func NewHubTestCleanCmd() *cobra.Command { func NewHubTestCleanCmd() *cobra.Command {
var cmdHubTestClean = &cobra.Command{ var cmdHubTestClean = &cobra.Command{
Use: "clean", Use: "clean",
@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command {
return cmdHubTestClean return cmdHubTestClean
} }
func NewHubTestInfoCmd() *cobra.Command { func NewHubTestInfoCmd() *cobra.Command {
var cmdHubTestInfo = &cobra.Command{ var cmdHubTestInfo = &cobra.Command{
Use: "info", Use: "info",
@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command {
return cmdHubTestInfo return cmdHubTestInfo
} }
func NewHubTestListCmd() *cobra.Command { func NewHubTestListCmd() *cobra.Command {
var cmdHubTestList = &cobra.Command{ var cmdHubTestList = &cobra.Command{
Use: "list", Use: "list",
@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command {
return cmdHubTestList return cmdHubTestList
} }
func NewHubTestCoverageCmd() *cobra.Command { func NewHubTestCoverageCmd() *cobra.Command {
var showParserCov bool var showParserCov bool
var showScenarioCov bool var showScenarioCov bool
@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
return fmt.Errorf("unable to load all tests: %+v", err) return fmt.Errorf("unable to load all tests: %+v", err)
} }
var err error var err error
scenarioCoverage := []hubtest.ScenarioCoverage{} scenarioCoverage := []hubtest.Coverage{}
parserCoverage := []hubtest.ParserCoverage{} parserCoverage := []hubtest.Coverage{}
scenarioCoveragePercent := 0 scenarioCoveragePercent := 0
parserCoveragePercent := 0 parserCoveragePercent := 0
@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
parserTested := 0 parserTested := 0
for _, test := range parserCoverage { for _, test := range parserCoverage {
if test.TestsCount > 0 { if test.TestsCount > 0 {
parserTested += 1 parserTested++
} }
} }
parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100))) parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
if err != nil { if err != nil {
return fmt.Errorf("while getting scenario coverage: %s", err) return fmt.Errorf("while getting scenario coverage: %s", err)
} }
scenarioTested := 0 scenarioTested := 0
for _, test := range scenarioCoverage { for _, test := range scenarioCoverage {
if test.TestsCount > 0 { if test.TestsCount > 0 {
scenarioTested += 1 scenarioTested++
} }
} }
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100))) scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
} }
@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
os.Exit(0) os.Exit(0)
} }
if csConfig.Cscli.Output == "human" { switch csConfig.Cscli.Output {
case "human":
if showParserCov || showAll { if showParserCov || showAll {
hubTestParserCoverageTable(color.Output, parserCoverage) hubTestParserCoverageTable(color.Output, parserCoverage)
} }
@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll { if showScenarioCov || showAll {
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent) fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
} }
} else if csConfig.Cscli.Output == "json" { case "json":
dump, err := json.MarshalIndent(parserCoverage, "", " ") dump, err := json.MarshalIndent(parserCoverage, "", " ")
if err != nil { if err != nil {
return err return err
@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
return err return err
} }
fmt.Printf("%s", dump) fmt.Printf("%s", dump)
} else { default:
return fmt.Errorf("only human/json output modes are supported") return fmt.Errorf("only human/json output modes are supported")
} }
return nil return nil
}, },
} }
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage") cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command {
return cmdHubTestCoverage return cmdHubTestCoverage
} }
func NewHubTestEvalCmd() *cobra.Command { func NewHubTestEvalCmd() *cobra.Command {
var evalExpression string var evalExpression string
var cmdHubTestEval = &cobra.Command{ var cmdHubTestEval = &cobra.Command{
@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command {
if err != nil { if err != nil {
return fmt.Errorf("can't load test: %+v", err) return fmt.Errorf("can't load test: %+v", err)
} }
err = test.ParserAssert.LoadTest(test.ParserResultFile) err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil { if err != nil {
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err) return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
} }
output, err := test.ParserAssert.EvalExpression(evalExpression) output, err := test.ParserAssert.EvalExpression(evalExpression)
if err != nil { if err != nil {
return err return err
} }
fmt.Print(output) fmt.Print(output)
} }
return nil return nil
}, },
} }
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval") cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmdHubTestEval return cmdHubTestEval
} }
func NewHubTestExplainCmd() *cobra.Command { func NewHubTestExplainCmd() *cobra.Command {
var cmdHubTestExplain = &cobra.Command{ var cmdHubTestExplain = &cobra.Command{
Use: "explain", Use: "explain",
@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command {
} }
err = test.ParserAssert.LoadTest(test.ParserResultFile) err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil { if err != nil {
err := test.Run() if err = test.Run(); err != nil {
if err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
} }
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil { if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
return fmt.Errorf("unable to load parser result after run: %s", err) return fmt.Errorf("unable to load parser result after run: %s", err)
} }
} }
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile) err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil { if err != nil {
err := test.Run() if err = test.Run(); err != nil {
if err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err) return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
} }
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil { if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
return fmt.Errorf("unable to load scenario result after run: %s", err) return fmt.Errorf("unable to load scenario result after run: %s", err)
} }
} }

10
cmd/crowdsec-cli/hubtest_table.go
View file

@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
t.Render() t.Render()
} }
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) { func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out) t := newLightTable(out)
t.SetHeaders("Parser", "Status", "Number of tests") t.SetHeaders("Parser", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0 parserTested := 0
for _, test := range coverage { for _, test := range coverage {
status := emoji.RedCircle.String() status := emoji.RedCircle.String()
if test.TestsCount > 0 { if test.TestsCount > 0 {
status = emoji.GreenCircle.String() status = emoji.GreenCircle.String()
parserTested++ parserTested++
} }
t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
} }
t.Render() t.Render()
} }
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) { func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out) t := newLightTable(out)
t.SetHeaders("Scenario", "Status", "Number of tests") t.SetHeaders("Scenario", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0 parserTested := 0
for _, test := range coverage { for _, test := range coverage {
status := emoji.RedCircle.String() status := emoji.RedCircle.String()
if test.TestsCount > 0 { if test.TestsCount > 0 {
status = emoji.GreenCircle.String() status = emoji.GreenCircle.String()
parserTested++ parserTested++
} }
t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn))) t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
} }
t.Render() t.Render()

236
cmd/crowdsec-cli/item_metrics.go Normal file
View file

@ -0,0 +1,236 @@
package main
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func ShowMetrics(hubItem *cwhub.Item) error {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, sub := range hubItem.SubItems() {
if err := ShowMetrics(sub); err != nil {
return err
}
}
default:
// no metrics for this item type
}
return nil
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

85
cmd/crowdsec-cli/item_suggest.go Normal file
View file

@ -0,0 +1,85 @@
package main
import (
"fmt"
"strings"
"github.com/agext/levenshtein"
"github.com/spf13/cobra"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
const MaxDistance = 7
// SuggestNearestMessage returns a message with the most similar item name, if one is found
func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
score := 100
nearest := ""
for _, item := range hub.GetItemMap(itemType) {
d := levenshtein.Distance(itemName, item.Name, nil)
if d < score {
score = d
nearest = item.Name
}
}
msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
if score < MaxDistance {
msg += fmt.Sprintf(", did you mean '%s'?", nearest)
}
return msg
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
for _, item := range hub.GetItemMap(itemType) {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := hub.GetInstalledItemNames(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}

606
cmd/crowdsec-cli/itemcommands.go Normal file
View file

@ -0,0 +1,606 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cmdHelp struct {
// Example is required, the others have a default value
// generated from the item type
use string
short string
long string
example string
}
type hubItemType struct {
name string // plural, as used in the hub index
singular string
oneOrMore string // parenthetical pluralizaion: "parser(s)"
help cmdHelp
installHelp cmdHelp
removeHelp cmdHelp
upgradeHelp cmdHelp
inspectHelp cmdHelp
listHelp cmdHelp
}
var hubItemTypes = map[string]hubItemType{
"parsers": {
name: "parsers",
singular: "parser",
oneOrMore: "parser(s)",
help: cmdHelp{
example: `cscli parsers list -a
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
`,
},
installHelp: cmdHelp{
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
removeHelp: cmdHelp{
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
upgradeHelp: cmdHelp{
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
inspectHelp: cmdHelp{
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
},
listHelp: cmdHelp{
example: `cscli parsers list
cscli parsers list -a
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
List only enabled parsers unless "-a" or names are specified.`,
},
},
"postoverflows": {
name: "postoverflows",
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cmdHelp{
example: `cscli postoverflows list -a
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
`,
},
installHelp: cmdHelp{
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
removeHelp: cmdHelp{
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
upgradeHelp: cmdHelp{
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
inspectHelp: cmdHelp{
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
listHelp: cmdHelp{
example: `cscli postoverflows list
cscli postoverflows list -a
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
List only enabled postoverflows unless "-a" or names are specified.`,
},
},
"scenarios": {
name: "scenarios",
singular: "scenario",
oneOrMore: "scenario(s)",
help: cmdHelp{
example: `cscli scenarios list -a
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
`,
},
installHelp: cmdHelp{
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
removeHelp: cmdHelp{
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
upgradeHelp: cmdHelp{
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
inspectHelp: cmdHelp{
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
listHelp: cmdHelp{
example: `cscli scenarios list
cscli scenarios list -a
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
List only enabled scenarios unless "-a" or names are specified.`,
},
},
"collections": {
name: "collections",
singular: "collection",
oneOrMore: "collection(s)",
help: cmdHelp{
example: `cscli collections list -a
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
`,
},
installHelp: cmdHelp{
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
},
removeHelp: cmdHelp{
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
},
upgradeHelp: cmdHelp{
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
},
inspectHelp: cmdHelp{
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
},
listHelp: cmdHelp{
example: `cscli collections list
cscli collections list -a
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
List only enabled collections unless "-a" or names are specified.`,
},
},
}
func NewItemsCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
Long: it.help.long,
Example: it.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{it.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(NewItemsInstallCmd(typeName))
cmd.AddCommand(NewItemsRemoveCmd(typeName))
cmd.AddCommand(NewItemsUpgradeCmd(typeName))
cmd.AddCommand(NewItemsInspectCmd(typeName))
cmd.AddCommand(NewItemsListCmd(typeName))
return cmd
}
func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
downloadOnly, err := flags.GetBool("download-only")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
ignoreError, err := flags.GetBool("ignore")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
msg := SuggestNearestMessage(hub, it.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
log.Errorf(msg)
continue
}
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
log.Infof(ReloadMessage())
return nil
}
return run
}
func NewItemsInstallCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.installHelp.use, "install [item]..."),
Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
Example: it.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(typeName, args, toComplete)
},
RunE: itemsInstallRunner(it),
}
flags := cmd.Flags()
flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
return cmd
}
// return the names of the installed parents of an item, used to check if we can remove it
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.Ancestors() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}
}
return ret
}
func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
purge, err := flags.GetBool("purge")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if all {
getter := hub.GetInstalledItems
if purge {
getter = hub.GetAllItems
}
items, err := getter(it.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
return run
}
func NewItemsRemoveCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.removeHelp.use, "remove [item]..."),
Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
Example: it.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsRemoveRunner(it),
}
flags := cmd.Flags()
flags.Bool("purge", false, "Delete source file too")
flags.Bool("force", false, "Force remove: remove tainted and outdated files")
flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
return cmd
}
func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if all {
items, err := hub.GetInstalledItems(it.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, it.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", item.Name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
return run
}
func NewItemsUpgradeCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
Example: it.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsUpgradeRunner(it),
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
url, err := flags.GetString("url")
if err != nil {
return err
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
noMetrics, err := flags.GetBool("no-metrics")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, it.name)
}
if err = InspectItem(item, !noMetrics); err != nil {
return err
}
}
return nil
}
return run
}
func NewItemsInspectCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
Example: it.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsInspectRunner(it),
}
flags := cmd.Flags()
flags.StringP("url", "u", "", "Prometheus url")
flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
items := make(map[string][]*cwhub.Item)
items[it.name], err = selectItems(hub, it.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{it.name}, items); err != nil {
return err
}
return nil
}
return run
}
func NewItemsListCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
Example: it.listHelp.example,
DisableAutoGenTag: true,
RunE: itemsListRunner(it),
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
return cmd
}

157
cmd/crowdsec-cli/items.go Normal file
View file

@ -0,0 +1,157 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"gopkg.in/yaml.v3"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
itemNames := hub.GetItemNames(itemType)
notExist := []string{}
if len(args) > 0 {
for _, arg := range args {
if !slices.Contains(itemNames, arg) {
notExist = append(notExist, arg)
}
}
}
if len(notExist) > 0 {
return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
}
if len(args) > 0 {
itemNames = args
installedOnly = false
}
items := make([]*cwhub.Item, 0, len(itemNames))
for _, itemName := range itemNames {
item := hub.GetItem(itemType, itemName)
if installedOnly && !item.State.Installed {
continue
}
items = append(items, item)
}
cwhub.SortItemSlice(items)
return items, nil
}
func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
switch csConfig.Cscli.Output {
case "human":
for _, itemType := range itemTypes {
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
}
case "json":
type itemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
hubStatus := make(map[string][]itemHubStatus)
for _, itemType := range itemTypes {
// empty slice in case there are no items of this type
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
for i, item := range items[itemType] {
status, emo := item.InstallStatus()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.State.LocalVersion,
LocalPath: item.State.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", emo, status),
}
}
}
x, err := json.MarshalIndent(hubStatus, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
out.Write(x)
case "raw":
csvwriter := csv.NewWriter(out)
header := []string{"name", "status", "version", "description"}
if len(itemTypes) > 1 {
header = append(header, "type")
}
if err := csvwriter.Write(header); err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, itemType := range itemTypes {
for _, item := range items[itemType] {
status, _ := item.InstallStatus()
row := []string{
item.Name,
status,
item.State.LocalVersion,
item.Description,
}
if len(itemTypes) > 1 {
row = append(row, itemType)
}
if err := csvwriter.Write(row); err != nil {
return fmt.Errorf("failed to write raw output: %s", err)
}
}
}
csvwriter.Flush()
default:
return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
}
return nil
}
func InspectItem(item *cwhub.Item, showMetrics bool) error {
switch csConfig.Cscli.Output {
case "human", "raw":
enc := yaml.NewEncoder(os.Stdout)
enc.SetIndent(2)
if err := enc.Encode(item); err != nil {
return fmt.Errorf("unable to encode item: %s", err)
}
case "json":
b, err := json.MarshalIndent(*item, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
fmt.Print(string(b))
}
if csConfig.Cscli.Output == "human" && showMetrics {
fmt.Printf("\nCurrent metrics: \n")
if err := ShowMetrics(item); err != nil {
return err
}
}
return nil
}

13
cmd/crowdsec-cli/lapi.go
View file

@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err) log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
} }
if err := require.Hub(csConfig); err != nil { hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err) log.Fatal(err)
} }
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil { if err != nil {
log.Fatalf("failed to get scenarios : %s", err) log.Fatalf("failed to get scenarios : %s", err)
} }
@ -338,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Fatalf("Failed to init expr helpers : %s", err) log.Fatalf("Failed to init expr helpers : %s", err)
} }
// Populate cwhub package tools hub, err := require.Hub(csConfig, nil)
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { if err != nil {
log.Fatalf("Failed to load hub index : %s", err) log.Fatal(err)
} }
csParsers := parser.NewParsers() csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil { if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err) log.Fatalf("unable to load parsers: %s", err)
} }

33
cmd/crowdsec-cli/main.go
View file

@ -4,7 +4,6 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"github.com/fatih/color" "github.com/fatih/color"
@ -12,9 +11,9 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/cobra/doc" "github.com/spf13/cobra/doc"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/fflag"
@ -29,15 +28,11 @@ var dbClient *database.Client
var OutputFormat string var OutputFormat string
var OutputColor string var OutputColor string
var downloadOnly bool
var forceAction bool
var purge bool
var all bool
var prometheusURL string
var mergedConfig string var mergedConfig string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
var flagBranch = ""
func initConfig() { func initConfig() {
var err error var err error
if trace_lvl { if trace_lvl {
@ -58,9 +53,6 @@ func initConfig() {
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatal(err)
}
} else { } else {
csConfig = csconfig.NewDefaultConfig() csConfig = csconfig.NewDefaultConfig()
} }
@ -71,13 +63,10 @@ func initConfig() {
log.Debugf("Enabled feature flags: %s", fflist) log.Debugf("Enabled feature flags: %s", fflist)
} }
if csConfig.Cscli == nil { if flagBranch != "" {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath) csConfig.Cscli.HubBranch = flagBranch
} }
if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
cwhub.HubBranch = csConfig.Cscli.HubBranch
}
if OutputFormat != "" { if OutputFormat != "" {
csConfig.Cscli.Output = OutputFormat csConfig.Cscli.Output = OutputFormat
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" { if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
@ -206,7 +195,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error") rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace") rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github") rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil { if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err) log.Fatalf("failed to hide flag: %s", err)
} }
@ -243,10 +232,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(NewSimulationCmds()) rootCmd.AddCommand(NewSimulationCmds())
rootCmd.AddCommand(NewBouncersCmd()) rootCmd.AddCommand(NewBouncersCmd())
rootCmd.AddCommand(NewMachinesCmd()) rootCmd.AddCommand(NewMachinesCmd())
rootCmd.AddCommand(NewParsersCmd())
rootCmd.AddCommand(NewScenariosCmd())
rootCmd.AddCommand(NewCollectionsCmd())
rootCmd.AddCommand(NewPostOverflowsCmd())
rootCmd.AddCommand(NewCapiCmd()) rootCmd.AddCommand(NewCapiCmd())
rootCmd.AddCommand(NewLapiCmd()) rootCmd.AddCommand(NewLapiCmd())
rootCmd.AddCommand(NewCompletionCmd()) rootCmd.AddCommand(NewCompletionCmd())
@ -255,6 +240,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(NewHubTestCmd()) rootCmd.AddCommand(NewHubTestCmd())
rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewNotificationsCmd())
rootCmd.AddCommand(NewSupportCmd()) rootCmd.AddCommand(NewSupportCmd())
rootCmd.AddCommand(NewItemsCmd("collections"))
rootCmd.AddCommand(NewItemsCmd("parsers"))
rootCmd.AddCommand(NewItemsCmd("scenarios"))
rootCmd.AddCommand(NewItemsCmd("postoverflows"))
if fflag.CscliSetup.IsEnabled() { if fflag.CscliSetup.IsEnabled() {
rootCmd.AddCommand(NewSetupCmd()) rootCmd.AddCommand(NewSetupCmd())

35
cmd/crowdsec-cli/metrics.go
View file

@ -284,8 +284,20 @@ var noUnit bool
func runMetrics(cmd *cobra.Command, args []string) error { func runMetrics(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadPrometheus(); err != nil { flags := cmd.Flags()
return fmt.Errorf("failed to load prometheus config: %w", err)
url, err := flags.GetString("url")
if err != nil {
return err
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
noUnit, err = flags.GetBool("no-unit")
if err != nil {
return err
} }
if csConfig.Prometheus == nil { if csConfig.Prometheus == nil {
@ -296,17 +308,8 @@ func runMetrics(cmd *cobra.Command, args []string) error {
return fmt.Errorf("prometheus is not enabled, can't show metrics") return fmt.Errorf("prometheus is not enabled, can't show metrics")
} }
if prometheusURL == "" { if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
prometheusURL = csConfig.Cscli.PrometheusUrl return err
}
if prometheusURL == "" {
return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
}
err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
if err != nil {
return fmt.Errorf("could not fetch prometheus metrics: %w", err)
} }
return nil return nil
} }
@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command {
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: runMetrics, RunE: runMetrics,
} }
cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units") flags := cmdMetrics.PersistentFlags()
flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
return cmdMetrics return cmdMetrics
} }

194
cmd/crowdsec-cli/parsers.go
View file

@ -1,194 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewParsersCmd() *cobra.Command {
var cmdParsers = &cobra.Command{
Use: "parsers [action] [config]",
Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
Example: `cscli parsers install crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/sshd-logs
cscli parsers list
cscli parsers remove crowdsecurity/sshd-logs
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"parser"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdParsers.AddCommand(NewParsersInstallCmd())
cmdParsers.AddCommand(NewParsersRemoveCmd())
cmdParsers.AddCommand(NewParsersUpgradeCmd())
cmdParsers.AddCommand(NewParsersInspectCmd())
cmdParsers.AddCommand(NewParsersListCmd())
return cmdParsers
}
func NewParsersInstallCmd() *cobra.Command {
var ignoreError bool
var cmdParsersInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given parser(s)",
Long: `Fetch and install given parser(s) from hub`,
Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS, name)
Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
return cmdParsersInstall
}
func NewParsersRemoveCmd() *cobra.Command {
cmdParsersRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given parser(s)",
Long: `Remove given parse(s) from hub`,
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one parser to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
}
return nil
},
}
cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
return cmdParsersRemove
}
func NewParsersUpgradeCmd() *cobra.Command {
cmdParsersUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given parser(s)",
Long: `Fetch and upgrade given parser(s) from hub`,
Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one parser to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
}
}
return nil
},
}
cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdParsersUpgrade
}
func NewParsersInspectCmd() *cobra.Command {
var cmdParsersInspect = &cobra.Command{
Use: "inspect [name]",
Short: "Inspect given parser",
Long: `Inspect given parser`,
Example: `cscli parsers inspect crowdsec/xxx`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS)
},
}
cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdParsersInspect
}
func NewParsersListCmd() *cobra.Command {
var cmdParsersList = &cobra.Command{
Use: "list [name]",
Short: "List all parsers or given one",
Long: `List all parsers or given one`,
Example: `cscli parsers list
cscli parser list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
},
}
cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdParsersList
}

191
cmd/crowdsec-cli/postoverflows.go
View file

@ -1,191 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewPostOverflowsCmd() *cobra.Command {
cmdPostOverflows := &cobra.Command{
Use: "postoverflows [action] [config]",
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
cscli postoverflows inspect crowdsecurity/cdn-whitelist
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
cscli postoverflows list
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
return cmdPostOverflows
}
func NewPostOverflowsInstallCmd() *cobra.Command {
var ignoreError bool
cmdPostOverflowsInstall := &cobra.Command{
Use: "install [config]",
Short: "Install given postoverflow(s)",
Long: `Fetch and install given postoverflow(s) from hub`,
Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
return cmdPostOverflowsInstall
}
func NewPostOverflowsRemoveCmd() *cobra.Command {
cmdPostOverflowsRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given postoverflow(s)",
Long: `remove given postoverflow(s)`,
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
}
return nil
},
}
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
return cmdPostOverflowsRemove
}
func NewPostOverflowsUpgradeCmd() *cobra.Command {
cmdPostOverflowsUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given postoverflow(s)",
Long: `Fetch and Upgrade given postoverflow(s) from hub`,
Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
}
}
return nil
},
}
cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdPostOverflowsUpgrade
}
func NewPostOverflowsInspectCmd() *cobra.Command {
cmdPostOverflowsInspect := &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given postoverflow",
Long: `Inspect given postoverflow`,
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS_OVFLW)
},
}
return cmdPostOverflowsInspect
}
func NewPostOverflowsListCmd() *cobra.Command {
cmdPostOverflowsList := &cobra.Command{
Use: "list [config]",
Short: "List all postoverflows or given one",
Long: `List all postoverflows or given one`,
Example: `cscli postoverflows list
cscli postoverflows list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
},
}
cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdPostOverflowsList
}

58
cmd/crowdsec-cli/require/branch.go Normal file
View file

@ -0,0 +1,58 @@
package require
// Set the appropriate hub branch according to config settings and crowdsec version
import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func chooseBranch(cfg *csconfig.Config) string {
// this was set from config.yaml or flag
if cfg.Cscli.HubBranch != "" {
log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
return cfg.Cscli.HubBranch
}
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using hub branch 'master'")
return "master"
}
log.Warnf("A new CrowdSec release is available (%s). "+
"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
latest, csVersion)
return csVersion
}
// HubBranch sets the branch (in cscli config) and returns its value
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
func HubBranch(cfg *csconfig.Config) string {
branch := chooseBranch(cfg)
cfg.Cscli.HubBranch = branch
return branch
}

44
cmd/crowdsec-cli/require/require.go
View file

@ -23,6 +23,7 @@ func CAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient == nil { if c.API.Server.OnlineClient == nil {
return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath) return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
} }
return nil return nil
} }
@ -30,6 +31,7 @@ func PAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials.PapiURL == "" { if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
return fmt.Errorf("no PAPI URL in configuration") return fmt.Errorf("no PAPI URL in configuration")
} }
return nil return nil
} }
@ -45,6 +47,7 @@ func DB(c *csconfig.Config) error {
if err := c.LoadDBConfig(); err != nil { if err := c.LoadDBConfig(); err != nil {
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err) return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
} }
return nil return nil
} }
@ -64,20 +67,33 @@ func Notifications(c *csconfig.Config) error {
return nil return nil
} }
func Hub (c *csconfig.Config) error { // RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
if err := c.LoadHub(); err != nil { func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
return err // set branch in config, and log if necessary
branch := HubBranch(c)
remote := &cwhub.RemoteHubCfg {
Branch: branch,
URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
IndexPath: ".index.json",
} }
if c.Hub == nil { return remote
return fmt.Errorf("you must configure cli before interacting with hub") }
}
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
cwhub.SetHubBranch() // If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
if err := cwhub.GetHubIdx(c.Hub); err != nil { local := c.Hub
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
} if local == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub")
return nil }
hub, err := cwhub.NewHub(local, remote, false)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
}
return hub, nil
} }

188
cmd/crowdsec-cli/scenarios.go
View file

@ -1,188 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewScenariosCmd() *cobra.Command {
var cmdScenarios = &cobra.Command{
Use: "scenarios [action] [config]",
Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
Example: `cscli scenarios list [-a]
cscli scenarios install crowdsecurity/ssh-bf
cscli scenarios inspect crowdsecurity/ssh-bf
cscli scenarios upgrade crowdsecurity/ssh-bf
cscli scenarios remove crowdsecurity/ssh-bf
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"scenario"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdScenarios.AddCommand(NewCmdScenariosInstall())
cmdScenarios.AddCommand(NewCmdScenariosRemove())
cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
cmdScenarios.AddCommand(NewCmdScenariosInspect())
cmdScenarios.AddCommand(NewCmdScenariosList())
return cmdScenarios
}
func NewCmdScenariosInstall() *cobra.Command {
var ignoreError bool
var cmdScenariosInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given scenario(s)",
Long: `Fetch and install given scenario(s) from hub`,
Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.SCENARIOS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
return cmdScenariosInstall
}
func NewCmdScenariosRemove() *cobra.Command {
var cmdScenariosRemove = &cobra.Command{
Use: "remove [config]",
Short: "Remove given scenario(s)",
Long: `remove given scenario(s)`,
Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one scenario to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
}
return nil
},
}
cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
return cmdScenariosRemove
}
func NewCmdScenariosUpgrade() *cobra.Command {
var cmdScenariosUpgrade = &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given scenario(s)",
Long: `Fetch and Upgrade given scenario(s) from hub`,
Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
}
}
return nil
},
}
cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdScenariosUpgrade
}
func NewCmdScenariosInspect() *cobra.Command {
var cmdScenariosInspect = &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given scenario",
Long: `Inspect given scenario`,
Example: `cscli scenarios inspect crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.SCENARIOS)
},
}
cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdScenariosInspect
}
func NewCmdScenariosList() *cobra.Command {
var cmdScenariosList = &cobra.Command{
Use: "list [config]",
Short: "List all scenario(s) or given one",
Long: `List all scenario(s) or given one`,
Example: `cscli scenarios list
cscli scenarios list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
},
}
cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdScenariosList
}

11
cmd/crowdsec-cli/setup.go
View file

@ -6,13 +6,15 @@ import (
"os" "os"
"os/exec" "os/exec"
goccyyaml "github.com/goccy/go-yaml"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
goccyyaml "github.com/goccy/go-yaml"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup" "github.com/crowdsecurity/crowdsec/pkg/setup"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
) )
// NewSetupCmd defines the "cscli setup" command. // NewSetupCmd defines the "cscli setup" command.
@ -303,7 +305,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return fmt.Errorf("while reading file %s: %w", fromFile, err) return fmt.Errorf("while reading file %s: %w", fromFile, err)
} }
if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil { hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
return err return err
} }

12
cmd/crowdsec-cli/simulation.go
View file

@ -3,11 +3,11 @@ package main
import ( import (
"fmt" "fmt"
"os" "os"
"slices"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -112,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
if err := csConfig.LoadSimulation(); err != nil { if err := csConfig.LoadSimulation(); err != nil {
log.Fatal(err) log.Fatal(err)
} }
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before using simulation")
}
if csConfig.Cscli.SimulationConfig == nil { if csConfig.Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured") return fmt.Errorf("no simulation configured")
} }
@ -145,18 +142,19 @@ func NewSimulationEnableCmd() *cobra.Command {
Example: `cscli simulation enable`, Example: `cscli simulation enable`,
DisableAutoGenTag: true, DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
if err := require.Hub(csConfig); err != nil { hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err) log.Fatal(err)
} }
if len(args) > 0 { if len(args) > 0 {
for _, scenario := range args { for _, scenario := range args {
var item = cwhub.GetItem(cwhub.SCENARIOS, scenario) var item = hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil { if item == nil {
log.Errorf("'%s' doesn't exist or is not a scenario", scenario) log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
continue continue
} }
if !item.Installed { if !item.State.Installed {
log.Warningf("'%s' isn't enabled", scenario) log.Warningf("'%s' isn't enabled", scenario)
} }
isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario) isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)

44
cmd/crowdsec-cli/support.go
View file

@ -58,10 +58,6 @@ func stripAnsiString(str string) string {
func collectMetrics() ([]byte, []byte, error) { func collectMetrics() ([]byte, []byte, error) {
log.Info("Collecting prometheus metrics") log.Info("Collecting prometheus metrics")
err := csConfig.LoadPrometheus()
if err != nil {
return nil, nil, err
}
if csConfig.Cscli.PrometheusUrl == "" { if csConfig.Cscli.PrometheusUrl == "" {
log.Warn("No Prometheus URL configured, metrics will not be collected") log.Warn("No Prometheus URL configured, metrics will not be collected")
@ -69,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) {
} }
humanMetrics := bytes.NewBuffer(nil) humanMetrics := bytes.NewBuffer(nil)
err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human") err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err) return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
} }
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil) req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err) return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
} }
@ -132,10 +128,21 @@ func collectOSInfo() ([]byte, error) {
return w.Bytes(), nil return w.Bytes(), nil
} }
func collectHubItems(itemType string) []byte { func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
var err error
out := bytes.NewBuffer(nil) out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType) log.Infof("Collecting %s list", itemType)
ListItems(out, []string{itemType}, []string{}, false, true, all)
items := make(map[string][]*cwhub.Item)
if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
if err := listItems(out, []string{itemType}, items); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes() return out.Bytes()
} }
@ -157,7 +164,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
return out.Bytes(), nil return out.Bytes(), nil
} }
func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte { func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil { if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?") return []byte("No agent credentials found, are we LAPI ?")
} }
@ -167,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil { if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err)) return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
} }
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil { if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err)) return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
} }
@ -295,7 +302,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
skipAgent = true skipAgent = true
} }
if err := require.Hub(csConfig); err != nil { hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error()) infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
@ -333,10 +341,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig() infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
if !skipHub { if !skipHub {
infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS) infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS) infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW) infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS) infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
} }
if !skipDB { if !skipDB {
@ -358,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login, infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
csConfig.API.Server.OnlineClient.Credentials.Password, csConfig.API.Server.OnlineClient.Credentials.Password,
csConfig.API.Server.OnlineClient.Credentials.URL, csConfig.API.Server.OnlineClient.Credentials.URL,
CAPIURLPrefix) CAPIURLPrefix,
hub)
} }
if !skipLAPI { if !skipLAPI {
@ -366,7 +375,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login, infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
csConfig.API.Client.Credentials.Password, csConfig.API.Client.Credentials.Password,
csConfig.API.Client.Credentials.URL, csConfig.API.Client.Credentials.URL,
LAPIURLPrefix) LAPIURLPrefix,
hub)
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile() infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
} }

437
cmd/crowdsec-cli/utils.go
View file

@ -1,36 +1,17 @@
package main package main
import ( import (
"encoding/csv"
"encoding/json"
"fmt" "fmt"
"io"
"math"
"net" "net"
"net/http"
"slices"
"strconv"
"strings" "strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/agext/levenshtein"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
) )
const MaxDistance = 7
func printHelp(cmd *cobra.Command) { func printHelp(cmd *cobra.Command) {
err := cmd.Help() err := cmd.Help()
if err != nil { if err != nil {
@ -38,197 +19,6 @@ func printHelp(cmd *cobra.Command) {
} }
} }
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
} else {
errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
}
if ignoreErr {
log.Error(errMsg)
} else {
log.Fatalf(errMsg)
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
for _, s := range allItems {
d := levenshtein.Distance(itemName, s, nil)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := cwhub.GetInstalledItemsAsString(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
for _, itemType := range itemTypes {
itemName := ""
if len(args) == 1 {
itemName = args[0]
}
hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
}
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
if err != nil {
log.Fatalf("failed to unmarshal")
}
out.Write(x)
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
if showHeader {
header := []string{"name", "status", "version", "description"}
if showType {
header = append(header, "type")
}
err := csvwriter.Write(header)
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
for _, status := range statuses {
if status.LocalVersion == "" {
status.LocalVersion = "n/a"
}
row := []string{
status.Name,
status.Status,
status.LocalVersion,
status.Description,
}
if showType {
row = append(row, itemType)
}
err := csvwriter.Write(row)
if err != nil {
log.Fatalf("failed to write raw output : %s", err)
}
}
}
csvwriter.Flush()
}
}
func InspectItem(name string, objecitemType string) {
hubItem := cwhub.GetItem(objecitemType, name)
if hubItem == nil {
log.Fatalf("unable to retrieve item.")
}
var b []byte
var err error
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
}
fmt.Printf("%s", string(b))
if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return
}
if prometheusURL == "" {
//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
listenAddr := csConfig.Prometheus.ListenAddr
if listenAddr == "" {
listenAddr = "127.0.0.1"
}
listenPort := csConfig.Prometheus.ListenPort
if listenPort == 0 {
listenPort = 6060
}
prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
log.Debugf("No prometheus URL provided using: %s", prometheusURL)
}
fmt.Printf("\nCurrent metrics : \n")
ShowMetrics(hubItem)
}
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error { func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
/*if a range is provided, change the scope*/ /*if a range is provided, change the scope*/
@ -259,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return nil return nil
} }
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(prometheusURL, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, item := range hubItem.Parsers {
metrics := GetParserMetric(prometheusURL, item)
parserMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Scenarios {
metrics := GetScenarioMetric(prometheusURL, item)
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
}
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}
func getDBClient() (*database.Client, error) { func getDBClient() (*database.Client, error) {
var err error var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
@ -518,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
} }
return slice return slice
} }

32
cmd/crowdsec-cli/utils_table.go
View file

@ -3,6 +3,7 @@ package main
import ( import (
"fmt" "fmt"
"io" "io"
"strconv"
"github.com/aquasecurity/table" "github.com/aquasecurity/table"
"github.com/enescakir/emoji" "github.com/enescakir/emoji"
@ -10,14 +11,15 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
) )
func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) { func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
t := newLightTable(out) t := newLightTable(out)
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path") t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, status := range statuses { for _, item := range items {
t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath) status, emo := item.InstallStatus()
t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.State.LocalVersion, item.State.LocalPath)
} }
renderTableTitle(out, title) renderTableTitle(out, title)
t.Render() t.Render()
@ -31,11 +33,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired") t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.AddRow( t.AddRow(
fmt.Sprintf("%d", metrics["curr_count"]), strconv.Itoa(metrics["curr_count"]),
fmt.Sprintf("%d", metrics["overflow"]), strconv.Itoa(metrics["overflow"]),
fmt.Sprintf("%d", metrics["instantiation"]), strconv.Itoa(metrics["instantiation"]),
fmt.Sprintf("%d", metrics["pour"]), strconv.Itoa(metrics["pour"]),
fmt.Sprintf("%d", metrics["underflow"]), strconv.Itoa(metrics["underflow"]),
) )
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName)) renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@ -43,23 +45,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
} }
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) { func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
skip := true
t := newTable(out) t := newTable(out)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed") t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
// don't show table if no hits
showTable := false
for source, stats := range metrics { for source, stats := range metrics {
if stats["hits"] > 0 { if stats["hits"] > 0 {
t.AddRow( t.AddRow(
source, source,
fmt.Sprintf("%d", stats["hits"]), strconv.Itoa(stats["hits"]),
fmt.Sprintf("%d", stats["parsed"]), strconv.Itoa(stats["parsed"]),
fmt.Sprintf("%d", stats["unparsed"]), strconv.Itoa(stats["unparsed"]),
) )
skip = false showTable = true
} }
} }
if !skip { if showTable {
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName)) renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
t.Render() t.Render()
} }

19
cmd/crowdsec/crowdsec.go
View file

@ -20,21 +20,16 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
) )
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error var err error
// Populate cwhub package tools
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
// Start loading configs // Start loading configs
csParsers := parser.NewParsers() csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, fmt.Errorf("while loading parsers: %w", err) return nil, fmt.Errorf("while loading parsers: %w", err)
} }
if err := LoadBuckets(cConfig); err != nil { if err := LoadBuckets(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading scenarios: %w", err) return nil, fmt.Errorf("while loading scenarios: %w", err)
} }
@ -44,7 +39,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
return csParsers, nil return csParsers, nil
} }
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
inputEventChan = make(chan types.Event) inputEventChan = make(chan types.Event)
inputLineChan = make(chan types.Event) inputLineChan = make(chan types.Event)
@ -99,7 +94,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ { for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error { outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput") defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil { if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
log.Fatalf("starting outputs error : %s", err) log.Fatalf("starting outputs error : %s", err)
return err return err
} }
@ -131,7 +126,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
return nil return nil
} }
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) { func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
crowdsecTomb.Go(func() error { crowdsecTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveCrowdsec") defer trace.CatchPanic("crowdsec/serveCrowdsec")
go func() { go func() {
@ -139,7 +134,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
// this logs every time, even at config reload // this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0)) log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true agentReady <- true
if err := runCrowdsec(cConfig, parsers); err != nil { if err := runCrowdsec(cConfig, parsers, hub); err != nil {
log.Fatalf("unable to start crowdsec routines: %s", err) log.Fatalf("unable to start crowdsec routines: %s", err)
} }
}() }()

21
cmd/crowdsec/main.go
View file

@ -75,20 +75,20 @@ type Flags struct {
type labelsMap map[string]string type labelsMap map[string]string
func LoadBuckets(cConfig *csconfig.Config) error { func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
var ( var (
err error err error
files []string files []string
) )
for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) { for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.Installed { if hubScenarioItem.State.Installed {
files = append(files, hubScenarioItem.LocalPath) files = append(files, hubScenarioItem.State.LocalPath)
} }
} }
buckets = leakybucket.NewBuckets() buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files)) log.Infof("Loading %d scenario files", len(files))
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent) holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
if err != nil { if err != nil {
return fmt.Errorf("scenario loading failed: %v", err) return fmt.Errorf("scenario loading failed: %v", err)
@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) { func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet) cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("while loading configuration file: %w", err)
}
if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
return nil, fmt.Errorf("unable to load configuration: common section is empty")
} }
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags) cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
dumpStates = true dumpStates = true
} }
// Configuration paths are dependency to load crowdsec configuration
if err := cConfig.LoadConfigurationPaths(); err != nil {
return nil, err
}
if flags.SingleFileType != "" && flags.OneShotDSN != "" { if flags.SingleFileType != "" && flags.OneShotDSN != "" {
// if we're in time-machine mode, we don't want to log to file // if we're in time-machine mode, we don't want to log to file
cConfig.Common.LogMedia = "stdout" cConfig.Common.LogMedia = "stdout"

8
cmd/crowdsec/metrics.go
View file

@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
if !config.Enabled { if !config.Enabled {
return return
} }
if config.ListenAddr == "" {
log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
config.ListenAddr = "127.0.0.1"
}
if config.ListenPort == 0 {
log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
config.ListenPort = 6060
}
// Registering prometheus // Registering prometheus
// If in aggregated mode, do not register events associated with a source, to keep the cardinality low // If in aggregated mode, do not register events associated with a source, to keep the cardinality low

7
cmd/crowdsec/output.go
View file

@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
var bucketOverflows []types.Event var bucketOverflows []types.Event
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets, func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error { postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
var err error var err error
ticker := time.NewTicker(1 * time.Second) ticker := time.NewTicker(1 * time.Second)
@ -70,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert var cache []types.RuntimeAlert
var cacheMutex sync.Mutex var cacheMutex sync.Mutex
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil { if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err) return fmt.Errorf("loading list of installed hub scenarios: %w", err)
} }
@ -93,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
URL: apiURL, URL: apiURL,
PapiURL: papiURL, PapiURL: papiURL,
VersionPrefix: "v1", VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)}, UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)},
}) })
if err != nil { if err != nil {
return fmt.Errorf("new client api: %w", err) return fmt.Errorf("new client api: %w", err)

19
cmd/crowdsec/serve.go
View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
} }
if !cConfig.DisableAgent { if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig) hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil { if err != nil {
return nil, fmt.Errorf("unable to init crowdsec: %w", err) return nil, fmt.Errorf("unable to init crowdsec: %w", err)
} }
@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
} }
agentReady := make(chan bool, 1) agentReady := make(chan bool, 1)
serveCrowdsec(csParsers, cConfig, agentReady) serveCrowdsec(csParsers, cConfig, hub, agentReady)
} }
log.Printf("Reload is finished") log.Printf("Reload is finished")
@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
} }
if !cConfig.DisableAgent { if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig) hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil { if err != nil {
return fmt.Errorf("crowdsec init: %w", err) return fmt.Errorf("crowdsec init: %w", err)
} }
// if it's just linting, we're done // if it's just linting, we're done
if !flags.TestMode { if !flags.TestMode {
serveCrowdsec(csParsers, cConfig, agentReady) serveCrowdsec(csParsers, cConfig, hub, agentReady)
} }
} else { } else {
agentReady <- true agentReady <- true

1
config/config.yaml
View file

@ -6,7 +6,6 @@ common:
log_max_size: 20 log_max_size: 20
compress_logs: true compress_logs: true
log_max_files: 10 log_max_files: 10
working_dir: .
config_paths: config_paths:
config_dir: /etc/crowdsec/ config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/ data_dir: /var/lib/crowdsec/data/

1
config/config_win.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file log_media: file
log_level: info log_level: info
log_dir: C:\ProgramData\CrowdSec\log\ log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths: config_paths:
config_dir: C:\ProgramData\CrowdSec\config\ config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\ data_dir: C:\ProgramData\CrowdSec\data\

1
config/config_win_no_lapi.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file log_media: file
log_level: info log_level: info
log_dir: C:\ProgramData\CrowdSec\log\ log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths: config_paths:
config_dir: C:\ProgramData\CrowdSec\config\ config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\ data_dir: C:\ProgramData\CrowdSec\data\

1
config/dev.yaml
View file

@ -2,7 +2,6 @@ common:
daemonize: true daemonize: true
log_media: stdout log_media: stdout
log_level: info log_level: info
working_dir: .
config_paths: config_paths:
config_dir: ./config config_dir: ./config
data_dir: ./data/ data_dir: ./data/

1
config/user.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout log_media: stdout
log_level: info log_level: info
log_dir: /var/log/ log_dir: /var/log/
working_dir: .
config_paths: config_paths:
config_dir: /etc/crowdsec/ config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data data_dir: /var/lib/crowdsec/data

1
docker/config.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout log_media: stdout
log_level: info log_level: info
log_dir: /var/log/ log_dir: /var/log/
working_dir: .
config_paths: config_paths:
config_dir: /etc/crowdsec/ config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/ data_dir: /var/lib/crowdsec/data/

22
docker/docker_start.sh
View file

@ -101,19 +101,23 @@ register_bouncer() {
# $2 can be install, remove, upgrade # $2 can be install, remove, upgrade
# $3 is a list of object names separated by space # $3 is a list of object names separated by space
cscli_if_clean() { cscli_if_clean() {
local itemtype="$1"
local action="$2"
local objs=$3
shift 3
# loop over all objects # loop over all objects
for obj in $3; do for obj in $objs; do
if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
echo "Object $1/$obj is tainted, skipping" echo "Object $itemtype/$obj is tainted, skipping"
else else
# # Too verbose? Only show errors if not in debug mode # # Too verbose? Only show errors if not in debug mode
# if [ "$DEBUG" != "true" ]; then # if [ "$DEBUG" != "true" ]; then
# error_only=--error # error_only=--error
# fi # fi
error_only="" error_only=""
echo "Running: cscli $error_only $1 $2 \"$obj\"" echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli $error_only "$1" "$2" "$obj" cscli $error_only "$itemtype" "$action" "$obj" "$@"
fi fi
done done
} }
@ -327,22 +331,22 @@ fi
## Remove collections, parsers, scenarios & postoverflows ## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then if [ "$DISABLE_COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli_if_clean collections remove "$DISABLE_COLLECTIONS" cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force
fi fi
if [ "$DISABLE_PARSERS" != "" ]; then if [ "$DISABLE_PARSERS" != "" ]; then
# shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli_if_clean parsers remove "$DISABLE_PARSERS" cscli_if_clean parsers remove "$DISABLE_PARSERS" --force
fi fi
if [ "$DISABLE_SCENARIOS" != "" ]; then if [ "$DISABLE_SCENARIOS" != "" ]; then
# shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force
fi fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
# shellcheck disable=SC2086 # shellcheck disable=SC2086
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
fi fi
## Register bouncers via env ## Register bouncers via env

8
docker/test/tests/test_hub_collections.py
View file

@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor):
cs.wait_for_log([ cs.wait_for_log([
# f'*collections install "{it1}"*' # f'*collections install "{it1}"*'
# f'*collections install "{it2}"*' # f'*collections install "{it2}"*'
f'*Enabled collections : {it1}*', f'*Enabled collections: {it1}*',
f'*Enabled collections : {it2}*', f'*Enabled collections: {it2}*',
]) ])
@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor):
assert it not in items assert it not in items
logs = cs.log_lines() logs = cs.log_lines()
# check that there was no attempt to install # check that there was no attempt to install
assert not any(f'Enabled collections : {it}' in line for line in logs) assert not any(f'Enabled collections: {it}' in line for line in logs)
# already done in bats, prividing here as example of a somewhat complex test # already done in bats, prividing here as example of a somewhat complex test
@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
# implicit check for tainted=False # implicit check for tainted=False
assert items[coll]['status'] == 'enabled' assert items[coll]['status'] == 'enabled'
cs.wait_for_log([ cs.wait_for_log([
f'*Enabled collections : {coll}*', f'*Enabled collections: {coll}*',
]) ])
scenario = 'crowdsecurity/http-crawl-non_statics' scenario = 'crowdsecurity/http-crawl-non_statics'

4
docker/test/tests/test_hub_scenarios.py
View file

@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor):
} }
with crowdsec(flavor=flavor, environment=env) as cs: with crowdsec(flavor=flavor, environment=env) as cs:
cs.wait_for_log([ cs.wait_for_log([
f'*scenarios install "{it1}*"', f'*scenarios install "{it1}"*',
f'*scenarios install "{it2}*"', f'*scenarios install "{it2}"*',
"*Starting processing data*" "*Starting processing data*"
]) ])
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)

2
go.mod
View file

@ -25,7 +25,7 @@ require (
github.com/c-robinson/iplib v1.0.3 github.com/c-robinson/iplib v1.0.3
github.com/cespare/xxhash/v2 v2.2.0 github.com/cespare/xxhash/v2 v2.2.0
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.4 github.com/crowdsecurity/go-cs-lib v0.0.5
github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/grokky v0.2.1
github.com/crowdsecurity/machineid v1.0.2 github.com/crowdsecurity/machineid v1.0.2
github.com/davecgh/go-spew v1.1.1 github.com/davecgh/go-spew v1.1.1

4
go.sum
View file

@ -140,8 +140,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims= github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

4
pkg/csconfig/api.go
View file

@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs)) log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
} }
if err := c.LoadCommon(); err != nil {
return fmt.Errorf("loading common configuration: %s", err)
}
c.API.Server.LogDir = c.Common.LogDir c.API.Server.LogDir = c.Common.LogDir
c.API.Server.LogMedia = c.Common.LogMedia c.API.Server.LogMedia = c.Common.LogMedia
c.API.Server.CompressLogs = c.Common.CompressLogs c.API.Server.CompressLogs = c.Common.CompressLogs

8
pkg/csconfig/api_test.go
View file

@ -3,7 +3,6 @@ package csconfig
import ( import (
"net" "net"
"os" "os"
"path/filepath"
"strings" "strings"
"testing" "testing"
@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) {
err := tmpLAPI.LoadProfiles() err := tmpLAPI.LoadProfiles()
require.NoError(t, err) require.NoError(t, err)
LogDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
logLevel := log.InfoLevel logLevel := log.InfoLevel
config := &Config{} config := &Config{}
fcontent, err := os.ReadFile("./testdata/config.yaml") fcontent, err := os.ReadFile("./testdata/config.yaml")
@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) {
DbPath: "./testdata/test.db", DbPath: "./testdata/test.db",
}, },
Common: &CommonCfg{ Common: &CommonCfg{
LogDir: "./testdata/", LogDir: "./testdata",
LogMedia: "stdout", LogMedia: "stdout",
}, },
DisableAPI: false, DisableAPI: false,
@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) {
ShareContext: ptr.Of(false), ShareContext: ptr.Of(false),
ConsoleManagement: ptr.Of(false), ConsoleManagement: ptr.Of(false),
}, },
LogDir: LogDirFullPath, LogDir: "./testdata",
LogMedia: "stdout", LogMedia: "stdout",
OnlineClient: &OnlineApiClientCfg{ OnlineClient: &OnlineApiClientCfg{
CredentialsFilePath: "./testdata/online-api-secrets.yaml", CredentialsFilePath: "./testdata/online-api-secrets.yaml",

11
pkg/csconfig/common.go
View file

@ -14,7 +14,7 @@ type CommonCfg struct {
LogMedia string `yaml:"log_media"` LogMedia string `yaml:"log_media"`
LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file
LogLevel *log.Level `yaml:"log_level"` LogLevel *log.Level `yaml:"log_level"`
WorkingDir string `yaml:"working_dir,omitempty"` ///var/run WorkingDir string `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
CompressLogs *bool `yaml:"compress_logs,omitempty"` CompressLogs *bool `yaml:"compress_logs,omitempty"`
LogMaxSize int `yaml:"log_max_size,omitempty"` LogMaxSize int `yaml:"log_max_size,omitempty"`
LogMaxAge int `yaml:"log_max_age,omitempty"` LogMaxAge int `yaml:"log_max_age,omitempty"`
@ -22,15 +22,18 @@ type CommonCfg struct {
ForceColorLogs bool `yaml:"force_color_logs,omitempty"` ForceColorLogs bool `yaml:"force_color_logs,omitempty"`
} }
func (c *Config) LoadCommon() error { func (c *Config) loadCommon() error {
var err error var err error
if c.Common == nil { if c.Common == nil {
return fmt.Errorf("no common block provided in configuration file") c.Common = &CommonCfg{}
}
if c.Common.LogMedia == "" {
c.Common.LogMedia = "stdout"
} }
var CommonCleanup = []*string{ var CommonCleanup = []*string{
&c.Common.LogDir, &c.Common.LogDir,
&c.Common.WorkingDir,
} }
for _, k := range CommonCleanup { for _, k := range CommonCleanup {
if *k == "" { if *k == "" {

83
pkg/csconfig/common_test.go
View file

@ -1,83 +0,0 @@
package csconfig
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadCommon(t *testing.T) {
pidDirPath := "./testdata"
LogDirFullPath, err := filepath.Abs("./testdata/log/")
require.NoError(t, err)
WorkingDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
tests := []struct {
name string
input *Config
expected *CommonCfg
expectedErr string
}{
{
name: "basic valid configuration",
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./testdata/log/",
WorkingDir: "./testdata/",
},
},
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
LogDir: LogDirFullPath,
WorkingDir: WorkingDirFullPath,
},
},
{
name: "empty working dir",
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./testdata/log/",
},
},
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
LogDir: LogDirFullPath,
},
},
{
name: "no common",
input: &Config{},
expected: nil,
expectedErr: "no common block provided in configuration file",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCommon()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
assert.Equal(t, tc.expected, tc.input.Common)
})
}
}

36
pkg/csconfig/config.go
View file

@ -36,7 +36,7 @@ type Config struct {
PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"` PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"`
DisableAPI bool `yaml:"-"` DisableAPI bool `yaml:"-"`
DisableAgent bool `yaml:"-"` DisableAgent bool `yaml:"-"`
Hub *Hub `yaml:"-"` Hub *LocalHubCfg `yaml:"-"`
} }
func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) { func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
@ -58,6 +58,37 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
// this is actually the "merged" yaml // this is actually the "merged" yaml
return nil, "", fmt.Errorf("%s: %w", configFile, err) return nil, "", fmt.Errorf("%s: %w", configFile, err)
} }
if cfg.Prometheus == nil {
cfg.Prometheus = &PrometheusCfg{}
}
if cfg.Prometheus.ListenAddr == "" {
cfg.Prometheus.ListenAddr = "127.0.0.1"
log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr)
}
if cfg.Prometheus.ListenPort == 0 {
cfg.Prometheus.ListenPort = 6060
log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
}
if err = cfg.loadCommon(); err != nil {
return nil, "", err
}
if err = cfg.loadConfigurationPaths(); err != nil {
return nil, "", err
}
if err = cfg.loadHub(); err != nil {
return nil, "", err
}
if err = cfg.loadCSCLI(); err != nil {
return nil, "", err
}
return &cfg, configData, nil return &cfg, configData, nil
} }
@ -65,11 +96,8 @@ func NewDefaultConfig() *Config {
logLevel := log.InfoLevel logLevel := log.InfoLevel
commonCfg := CommonCfg{ commonCfg := CommonCfg{
Daemonize: false, Daemonize: false,
PidDir: "/tmp/",
LogMedia: "stdout", LogMedia: "stdout",
//LogDir unneeded
LogLevel: &logLevel, LogLevel: &logLevel,
WorkingDir: ".",
} }
prometheus := PrometheusCfg{ prometheus := PrometheusCfg{
Enabled: true, Enabled: true,

2
pkg/csconfig/config_paths.go
View file

@ -15,7 +15,7 @@ type ConfigurationPaths struct {
NotificationDir string `yaml:"notification_dir,omitempty"` NotificationDir string `yaml:"notification_dir,omitempty"`
} }
func (c *Config) LoadConfigurationPaths() error { func (c *Config) loadConfigurationPaths() error {
var err error var err error
if c.ConfigPaths == nil { if c.ConfigPaths == nil {
return fmt.Errorf("no configuration paths provided") return fmt.Errorf("no configuration paths provided")

4
pkg/csconfig/config_test.go
View file

@ -15,10 +15,10 @@ func TestNormalLoad(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false) _, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage) require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false) _, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config") require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
} }
func TestNewCrowdSecConfig(t *testing.T) { func TestNewCrowdSecConfig(t *testing.T) {

15
pkg/csconfig/crowdsec_service.go
View file

@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct {
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"` SimulationFilePath string `yaml:"-"`
ContextToSend map[string][]string `yaml:"-"` ContextToSend map[string][]string `yaml:"-"`
} }
@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error {
return fmt.Errorf("load error (simulation): %w", err) return fmt.Errorf("load error (simulation): %w", err)
} }
c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
c.Crowdsec.DataDir = c.ConfigPaths.DataDir
c.Crowdsec.HubDir = c.ConfigPaths.HubDir
c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
if c.Crowdsec.ParserRoutinesCount <= 0 { if c.Crowdsec.ParserRoutinesCount <= 0 {
c.Crowdsec.ParserRoutinesCount = 1 c.Crowdsec.ParserRoutinesCount = 1
} }
@ -145,15 +136,11 @@ func (c *Config) LoadCrowdsec() error {
return fmt.Errorf("loading api client: %s", err) return fmt.Errorf("loading api client: %s", err)
} }
if err := c.LoadHub(); err != nil {
return fmt.Errorf("while loading hub: %w", err)
}
c.Crowdsec.ContextToSend = make(map[string][]string, 0) c.Crowdsec.ContextToSend = make(map[string][]string, 0)
fallback := false fallback := false
if c.Crowdsec.ConsoleContextPath == "" { if c.Crowdsec.ConsoleContextPath == "" {
// fallback to default config file // fallback to default config file
c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml") c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
fallback = true fallback = true
} }

26
pkg/csconfig/crowdsec_service_test.go
View file

@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) {
acquisDirFullPath, err := filepath.Abs("./testdata/acquis") acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
require.NoError(t, err) require.NoError(t, err)
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
contextFileFullPath, err := filepath.Abs("./testdata/context.yaml") contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
require.NoError(t, err) require.NoError(t, err)
@ -66,10 +54,6 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: "", AcquisitionDirPath: "",
ConsoleContextPath: contextFileFullPath, ConsoleContextPath: contextFileFullPath,
AcquisitionFilePath: acquisFullPath, AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
BucketsRoutinesCount: 1, BucketsRoutinesCount: 1,
ParserRoutinesCount: 1, ParserRoutinesCount: 1,
OutputRoutinesCount: 1, OutputRoutinesCount: 1,
@ -109,10 +93,6 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: acquisDirFullPath, AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath, AcquisitionFilePath: acquisFullPath,
ConsoleContextPath: contextFileFullPath, ConsoleContextPath: contextFileFullPath,
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
BucketsRoutinesCount: 1, BucketsRoutinesCount: 1,
ParserRoutinesCount: 1, ParserRoutinesCount: 1,
OutputRoutinesCount: 1, OutputRoutinesCount: 1,
@ -141,7 +121,7 @@ func TestLoadCrowdsec(t *testing.T) {
}, },
}, },
Crowdsec: &CrowdsecServiceCfg{ Crowdsec: &CrowdsecServiceCfg{
ConsoleContextPath: contextFileFullPath, ConsoleContextPath: "./testdata/context.yaml",
ConsoleContextValueLength: 10, ConsoleContextValueLength: 10,
}, },
}, },
@ -149,10 +129,6 @@ func TestLoadCrowdsec(t *testing.T) {
Enable: ptr.Of(true), Enable: ptr.Of(true),
AcquisitionDirPath: "", AcquisitionDirPath: "",
AcquisitionFilePath: "", AcquisitionFilePath: "",
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
ConsoleContextPath: contextFileFullPath, ConsoleContextPath: contextFileFullPath,
BucketsRoutinesCount: 1, BucketsRoutinesCount: 1,
ParserRoutinesCount: 1, ParserRoutinesCount: 1,

20
pkg/csconfig/cscli.go
View file

@ -1,5 +1,9 @@
package csconfig package csconfig
import (
"fmt"
)
/*cscli specific config, such as hub directory*/ /*cscli specific config, such as hub directory*/
type CscliCfg struct { type CscliCfg struct {
Output string `yaml:"output,omitempty"` Output string `yaml:"output,omitempty"`
@ -7,25 +11,19 @@ type CscliCfg struct {
HubBranch string `yaml:"hub_branch"` HubBranch string `yaml:"hub_branch"`
SimulationConfig *SimulationConfig `yaml:"-"` SimulationConfig *SimulationConfig `yaml:"-"`
DbConfig *DatabaseCfg `yaml:"-"` DbConfig *DatabaseCfg `yaml:"-"`
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"` SimulationFilePath string `yaml:"-"`
PrometheusUrl string `yaml:"prometheus_uri"` PrometheusUrl string `yaml:"prometheus_uri"`
} }
func (c *Config) LoadCSCLI() error { func (c *Config) loadCSCLI() error {
if c.Cscli == nil { if c.Cscli == nil {
c.Cscli = &CscliCfg{} c.Cscli = &CscliCfg{}
} }
if err := c.LoadConfigurationPaths(); err != nil {
return err if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
} }
c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
c.Cscli.DataDir = c.ConfigPaths.DataDir
c.Cscli.HubDir = c.ConfigPaths.HubDir
c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
return nil return nil
} }

33
pkg/csconfig/cscli_test.go
View file

@ -1,28 +1,14 @@
package csconfig package csconfig
import ( import (
"path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest" "github.com/crowdsecurity/go-cs-lib/cstest"
) )
func TestLoadCSCLI(t *testing.T) { func TestLoadCSCLI(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
input *Config input *Config
@ -38,26 +24,23 @@ func TestLoadCSCLI(t *testing.T) {
HubDir: "./hub", HubDir: "./hub",
HubIndexFile: "./hub/.index.json", HubIndexFile: "./hub/.index.json",
}, },
Prometheus: &PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
},
}, },
expected: &CscliCfg{ expected: &CscliCfg{
ConfigDir: configDirFullPath, PrometheusUrl: "http://127.0.0.1:6060/metrics",
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
}, },
}, },
{
name: "no configuration path",
input: &Config{},
expected: &CscliCfg{},
expectedErr: "no configuration paths provided",
},
} }
for _, tc := range tests { for _, tc := range tests {
tc := tc tc := tc
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCSCLI() err := tc.input.loadCSCLI()
cstest.RequireErrorContains(t, err, tc.expectedErr) cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" { if tc.expectedErr != "" {
return return

20
pkg/csconfig/hub.go
View file

@ -1,19 +1,15 @@
package csconfig package csconfig
/*cscli specific config, such as hub directory*/ // LocalHubCfg holds the configuration for a local hub: where to download etc.
type Hub struct { type LocalHubCfg struct {
HubIndexFile string HubIndexFile string // Path to the local index file
HubDir string HubDir string // Where the hub items are downloaded
InstallDir string InstallDir string // Where to install items
InstallDataDir string InstallDataDir string // Where to install data
} }
func (c *Config) LoadHub() error { func (c *Config) loadHub() error {
if err := c.LoadConfigurationPaths(); err != nil { c.Hub = &LocalHubCfg{
return err
}
c.Hub = &Hub{
HubIndexFile: c.ConfigPaths.HubIndexFile, HubIndexFile: c.ConfigPaths.HubIndexFile,
HubDir: c.ConfigPaths.HubDir, HubDir: c.ConfigPaths.HubDir,
InstallDir: c.ConfigPaths.ConfigDir, InstallDir: c.ConfigPaths.ConfigDir,

40
pkg/csconfig/hub_test.go
View file

@ -1,32 +1,18 @@
package csconfig package csconfig
import ( import (
"path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest" "github.com/crowdsecurity/go-cs-lib/cstest"
) )
func TestLoadHub(t *testing.T) { func TestLoadHub(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
input *Config input *Config
expected *Hub expected *LocalHubCfg
expectedErr string expectedErr string
}{ }{
{ {
@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) {
HubIndexFile: "./hub/.index.json", HubIndexFile: "./hub/.index.json",
}, },
}, },
expected: &Hub{ expected: &LocalHubCfg{
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
InstallDir: configDirFullPath,
InstallDataDir: dataFullPath,
},
},
{
name: "no data dir",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./testdata",
HubDir: "./hub", HubDir: "./hub",
HubIndexFile: "./hub/.index.json", HubIndexFile: "./hub/.index.json",
InstallDir: "./testdata",
InstallDataDir: "./data",
}, },
}, },
expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
},
{
name: "no configuration path",
input: &Config{},
expectedErr: "no configuration paths provided",
},
} }
for _, tc := range tests { for _, tc := range tests {
tc := tc tc := tc
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadHub() err := tc.input.loadHub()
cstest.RequireErrorContains(t, err, tc.expectedErr) cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" { if tc.expectedErr != "" {
return return

11
pkg/csconfig/prometheus.go
View file

@ -1,19 +1,8 @@
package csconfig package csconfig
import "fmt"
type PrometheusCfg struct { type PrometheusCfg struct {
Enabled bool `yaml:"enabled"` Enabled bool `yaml:"enabled"`
Level string `yaml:"level"` //aggregated|full Level string `yaml:"level"` //aggregated|full
ListenAddr string `yaml:"listen_addr"` ListenAddr string `yaml:"listen_addr"`
ListenPort int `yaml:"listen_port"` ListenPort int `yaml:"listen_port"`
} }
func (c *Config) LoadPrometheus() error {
if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil {
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
}
}
return nil
}

42
pkg/csconfig/prometheus_test.go
View file

@ -1,42 +0,0 @@
package csconfig
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadPrometheus(t *testing.T) {
tests := []struct {
name string
input *Config
expectedURL string
expectedErr string
}{
{
name: "basic valid configuration",
input: &Config{
Prometheus: &PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
},
Cscli: &CscliCfg{},
},
expectedURL: "http://127.0.0.1:6060",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadPrometheus()
cstest.RequireErrorContains(t, err, tc.expectedErr)
require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl)
})
}
}

5
pkg/csconfig/simulation.go
View file

@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
} }
func (c *Config) LoadSimulation() error { func (c *Config) LoadSimulation() error {
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
simCfg := SimulationConfig{} simCfg := SimulationConfig{}
if c.ConfigPaths.SimulationFilePath == "" { if c.ConfigPaths.SimulationFilePath == "" {
c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml") c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")

13
pkg/csconfig/simulation_test.go
View file

@ -2,7 +2,6 @@ package csconfig
import ( import (
"fmt" "fmt"
"path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -12,12 +11,6 @@ import (
) )
func TestSimulationLoading(t *testing.T) { func TestSimulationLoading(t *testing.T) {
testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml")
require.NoError(t, err)
badYamlFullPath, err := filepath.Abs("./testdata/config.yaml")
require.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
input *Config input *Config
@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) {
}, },
Crowdsec: &CrowdsecServiceCfg{}, Crowdsec: &CrowdsecServiceCfg{},
}, },
expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage), expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage),
}, },
{ {
name: "basic bad file content", name: "basic bad file content",
@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) {
}, },
Crowdsec: &CrowdsecServiceCfg{}, Crowdsec: &CrowdsecServiceCfg{},
}, },
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
}, },
{ {
name: "basic bad file content", name: "basic bad file content",
@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) {
}, },
Crowdsec: &CrowdsecServiceCfg{}, Crowdsec: &CrowdsecServiceCfg{},
}, },
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath), expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
}, },
} }

1
pkg/csconfig/testdata/config.yaml vendored
View file

@ -2,7 +2,6 @@ common:
daemonize: false daemonize: false
log_media: stdout log_media: stdout
log_level: info log_level: info
working_dir: .
prometheus: prometheus:
enabled: true enabled: true
level: full level: full

284
pkg/cwhub/cwhub.go
View file

@ -2,287 +2,31 @@ package cwhub
import ( import (
"fmt" "fmt"
"os" "net/http"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"time"
"github.com/enescakir/emoji"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
) )
const ( var hubClient = &http.Client{
HubIndexFile = ".index.json" Timeout: 120 * time.Second,
// managed item types
PARSERS = "parsers"
PARSERS_OVFLW = "postoverflows"
SCENARIOS = "scenarios"
COLLECTIONS = "collections"
)
var (
ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS}
ErrMissingReference = errors.New("Reference(s) missing in collection")
// XXX: can we remove these globals?
skippedLocal = 0
skippedTainted = 0
RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
HubBranch = "master"
hubIdx map[string]map[string]Item
)
type ItemVersion struct {
Digest string `json:"digest,omitempty"` // meow
Deprecated bool `json:"deprecated,omitempty"`
} }
type ItemHubStatus struct { // safePath returns a joined path and ensures that it does not escape the base directory.
Name string `json:"name"` func safePath(dir, filePath string) (string, error) {
LocalVersion string `json:"local_version"` absBaseDir, err := filepath.Abs(filepath.Clean(dir))
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
// Item can be: parser, scenario, collection..
type Item struct {
// descriptive info
Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich)
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name"
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .config.json
Author string `json:"author,omitempty"` // as seen in .config.json
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .config.json
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
// remote (hub) info
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty"` // the last version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
// local (deployed) info
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` // the local meow
Installed bool `json:"installed,omitempty"`
Downloaded bool `json:"downloaded,omitempty"`
UpToDate bool `json:"up_to_date,omitempty"`
Tainted bool `json:"tainted,omitempty"` // has it been locally modified
Local bool `json:"local,omitempty"` // if it's a non versioned control one
// if it's a collection, it's not a single file
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
func (i *Item) status() (string, emoji.Emoji) {
status := "disabled"
ok := false
if i.Installed {
ok = true
status = "enabled"
}
managed := true
if i.Local {
managed = false
status += ",local"
}
warning := false
if i.Tainted {
warning = true
status += ",tainted"
} else if !i.UpToDate && !i.Local {
warning = true
status += ",update-available"
}
emo := emoji.QuestionMark
switch {
case !managed:
emo = emoji.House
case !i.Installed:
emo = emoji.Prohibited
case warning:
emo = emoji.Warning
case ok:
emo = emoji.CheckMark
}
return status, emo
}
func (i *Item) hubStatus() ItemHubStatus {
status, emo := i.status()
return ItemHubStatus{
Name: i.Name,
LocalVersion: i.LocalVersion,
LocalPath: i.LocalPath,
Description: i.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", emo, status),
}
}
// versionStatus: semver requires 'v' prefix
func (i *Item) versionStatus() int {
return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
}
func GetItemMap(itemType string) map[string]Item {
m, ok := hubIdx[itemType]
if !ok {
return nil
}
return m
}
// Given a FileInfo, extract the map key. Follow a symlink if necessary
func itemKey(itemPath string) (string, error) {
f, err := os.Lstat(itemPath)
if err != nil { if err != nil {
return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err) return "", err
} }
if f.Mode()&os.ModeSymlink == 0 { absFilePath, err := filepath.Abs(filepath.Join(dir, filePath))
// it's not a symlink, so the filename itsef should be the key
return filepath.Base(itemPath), nil
}
// resolve the symlink to hub file
pathInHub, err := os.Readlink(itemPath)
if err != nil { if err != nil {
return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err) return "", err
} }
author := filepath.Base(filepath.Dir(pathInHub)) if !strings.HasPrefix(absFilePath, absBaseDir) {
return "", fmt.Errorf("path %s escapes base directory %s", filePath, dir)
}
fname := filepath.Base(pathInHub) return absFilePath, nil
fname = strings.TrimSuffix(fname, ".yaml")
fname = strings.TrimSuffix(fname, ".yml")
return fmt.Sprintf("%s/%s", author, fname), nil
}
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
itemKey, err := itemKey(itemPath)
if err != nil {
return nil, err
}
m := GetItemMap(itemType)
if m == nil {
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
}
v, ok := m[itemKey]
if !ok {
return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
}
return &v, nil
}
func GetItem(itemType string, itemName string) *Item {
if m, ok := GetItemMap(itemType)[itemName]; ok {
return &m
}
return nil
}
func AddItem(itemType string, item Item) error {
for _, itype := range ItemTypes {
if itype == itemType {
hubIdx[itemType][item.Name] = item
return nil
}
}
return fmt.Errorf("ItemType %s is unknown", itemType)
}
func DisplaySummary() {
log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
if skippedLocal > 0 || skippedTainted > 0 {
log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
}
}
func GetInstalledItems(itemType string) ([]Item, error) {
items, ok := hubIdx[itemType]
if !ok {
return nil, fmt.Errorf("no %s in hubIdx", itemType)
}
retItems := make([]Item, 0)
for _, item := range items {
if item.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
func GetInstalledItemsAsString(itemType string) ([]string, error) {
items, err := GetInstalledItems(itemType)
if err != nil {
return nil, err
}
retStr := make([]string, len(items))
for i, it := range items {
retStr[i] = it.Name
}
return retStr, nil
}
// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
if _, ok := hubIdx[itemType]; !ok {
log.Errorf("type %s doesn't exist", itemType)
return nil
}
ret := make([]ItemHubStatus, 0)
// remember, you do it for the user :)
for _, item := range hubIdx[itemType] {
if name != "" && name != item.Name {
// user has requested a specific name
continue
}
// Only enabled items ?
if !all && !item.Installed {
continue
}
// Check the item status
ret = append(ret, item.hubStatus())
}
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
return ret
} }

326
pkg/cwhub/cwhub_test.go
View file

@ -9,14 +9,13 @@ import (
"testing" "testing"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
) )
const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
/* /*
To test : To test :
- Download 'first' hub index - Download 'first' hub index
@ -28,294 +27,63 @@ import (
var responseByPath map[string]string var responseByPath map[string]string
func TestItemStatus(t *testing.T) { // testHub initializes a temporary hub with an empty json file, optionally updating it.
cfg := envSetup(t) func testHub(t *testing.T, update bool) *Hub {
defer envTearDown(cfg) tmpDir, err := os.MkdirTemp("", "testhub")
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub)
require.NoError(t, err, "failed to load hub index")
// get existing map
x := GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item : good and bad
for k := range x {
item := GetItem(COLLECTIONS, k)
require.NotNil(t, item)
item.Installed = true
item.UpToDate = false
item.Local = false
item.Tainted = false
txt, _ := item.status()
require.Equal(t, "enabled,update-available", txt)
item.Installed = false
item.UpToDate = false
item.Local = true
item.Tainted = false
txt, _ = item.status()
require.Equal(t, "disabled,local", txt)
}
DisplaySummary()
}
func TestGetters(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub)
require.NoError(t, err, "failed to load hub index")
// get non existing map
empty := GetItemMap("ratata")
require.Nil(t, empty)
// get existing map
x := GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item : good and bad
for k := range x {
empty := GetItem(COLLECTIONS, k+"nope")
require.Nil(t, empty)
item := GetItem(COLLECTIONS, k)
require.NotNil(t, item)
// Add item and get it
item.Name += "nope"
err := AddItem(COLLECTIONS, *item)
require.NoError(t, err) require.NoError(t, err)
newitem := GetItem(COLLECTIONS, item.Name) local := &csconfig.LocalHubCfg{
require.NotNil(t, newitem) HubDir: filepath.Join(tmpDir, "crowdsec", "hub"),
HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"),
err = AddItem("ratata", *item) InstallDir: filepath.Join(tmpDir, "crowdsec"),
cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") InstallDataDir: filepath.Join(tmpDir, "installed-data"),
} }
}
func TestIndexDownload(t *testing.T) { err = os.MkdirAll(local.HubDir, 0o700)
cfg := envSetup(t) require.NoError(t, err)
defer envTearDown(cfg)
// DownloadHubIdx() err = os.MkdirAll(local.InstallDir, 0o700)
err := UpdateHubIdx(cfg.Hub) require.NoError(t, err)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub) err = os.MkdirAll(local.InstallDataDir, 0o700)
require.NoError(t, err, "failed to load hub index") require.NoError(t, err)
}
func getTestCfg() *csconfig.Config { err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644)
cfg := &csconfig.Config{Hub: &csconfig.Hub{}} require.NoError(t, err)
cfg.Hub.InstallDir, _ = filepath.Abs("./install")
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
return cfg
}
func envSetup(t *testing.T) *csconfig.Config {
resetResponseByPath()
log.SetLevel(log.DebugLevel)
cfg := getTestCfg()
defaultTransport := http.DefaultClient.Transport
t.Cleanup(func() { t.Cleanup(func() {
http.DefaultClient.Transport = defaultTransport os.RemoveAll(tmpDir)
})
remote := &RemoteHubCfg{
Branch: "master",
URLTemplate: mockURLTemplate,
IndexPath: ".index.json",
}
hub, err := NewHub(local, remote, update)
require.NoError(t, err)
return hub
}
// envSetup initializes the temporary hub and mocks the http client.
func envSetup(t *testing.T) *Hub {
setResponseByPath()
log.SetLevel(log.DebugLevel)
defaultTransport := hubClient.Transport
t.Cleanup(func() {
hubClient.Transport = defaultTransport
}) })
// Mock the http client // Mock the http client
http.DefaultClient.Transport = newMockTransport() hubClient.Transport = newMockTransport()
err := os.MkdirAll(cfg.Hub.InstallDir, 0700) hub := testHub(t, true)
require.NoError(t, err)
err = os.MkdirAll(cfg.Hub.HubDir, 0700) return hub
require.NoError(t, err)
err = UpdateHubIdx(cfg.Hub)
require.NoError(t, err)
// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
// log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
// }
// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
// log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
// }
return cfg
}
func envTearDown(cfg *csconfig.Config) {
if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
}
if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
}
}
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
// Install the parser
err := DownloadLatest(cfg, &item, false, false)
require.NoError(t, err, "failed to download %s", item.Name)
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
err = EnableItem(cfg, &item)
require.NoError(t, err, "failed to enable %s", item.Name)
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
}
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
defer f.Close()
_, err = f.WriteString("tainted")
require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
// Local sync and check status
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
}
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
err := DownloadLatest(cfg, &item, true, true)
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
// Remove
err := DisableItem(cfg, &item, false, false)
require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status
warns, err := LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings : %+v", warns)
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
// Purge
err = DisableItem(cfg, &item, true, false)
require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status
warns, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings : %+v", warns)
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
}
func TestInstallParser(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
cfg := envSetup(t)
defer envTearDown(cfg)
getHubIdxOrFail(t)
// map iteration is random by itself
for _, it := range hubIdx[PARSERS] {
testInstallItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
testTaintItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
testDisableItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
break
}
}
func TestInstallCollection(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
cfg := envSetup(t)
defer envTearDown(cfg)
getHubIdxOrFail(t)
// map iteration is random by itself
for _, it := range hubIdx[COLLECTIONS] {
testInstallItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testTaintItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testDisableItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
log.Infof("%+v", x)
break
}
} }
type mockTransport struct{} type mockTransport struct{}
@ -324,7 +92,7 @@ func newMockTransport() http.RoundTripper {
return &mockTransport{} return &mockTransport{}
} }
// Implement http.RoundTripper // Implement http.RoundTripper.
func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// Create mocked http.Response // Create mocked http.Response
response := &http.Response{ response := &http.Response{
@ -362,7 +130,7 @@ func fileToStringX(path string) string {
return strings.ReplaceAll(string(data), "\r\n", "\n") return strings.ReplaceAll(string(data), "\r\n", "\n")
} }
func resetResponseByPath() { func setResponseByPath() {
responseByPath = map[string]string{ responseByPath = map[string]string{
"/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),
"/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"), "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),

72
pkg/cwhub/dataset.go
View file

@ -1,70 +1,84 @@
package cwhub package cwhub
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"os" "os"
"path/filepath"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
) )
// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
type DataSet struct { type DataSet struct {
Data []*types.DataSource `yaml:"data,omitempty"` Data []types.DataSource `yaml:"data,omitempty"`
} }
// downloadFile downloads a file and writes it to disk, with no hash verification.
func downloadFile(url string, destPath string) error { func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath) log.Debugf("downloading %s in %s", url, destPath)
req, err := http.NewRequest(http.MethodGet, url, nil) resp, err := hubClient.Get(url)
if err != nil { if err != nil {
return err return fmt.Errorf("while downloading %s: %w", url, err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
} }
defer resp.Body.Close() defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body)) return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
} }
file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) file, err := os.Create(destPath)
if err != nil {
return err
}
defer file.Close()
// avoid reading the whole file in memory
_, err = io.Copy(file, resp.Body)
if err != nil { if err != nil {
return err return err
} }
_, err = file.Write(body) if err = file.Sync(); err != nil {
if err != nil {
return err
}
err = file.Sync()
if err != nil {
return err return err
} }
return nil return nil
} }
func GetData(data []*types.DataSource, dataDir string) error { // downloadDataSet downloads all the data files for an item.
for _, dataS := range data { func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
destPath := filepath.Join(dataDir, dataS.DestPath) dec := yaml.NewDecoder(reader)
for {
data := &DataSet{}
if err := dec.Decode(data); err != nil {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("while reading file: %w", err)
}
for _, dataS := range data.Data {
destPath, err := safePath(dataFolder, dataS.DestPath)
if err != nil {
return err
}
if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
err := downloadFile(dataS.SourceURL, destPath) if err := downloadFile(dataS.SourceURL, destPath); err != nil {
if err != nil { return fmt.Errorf("while getting data: %w", err)
return err }
}
} }
} }

17
pkg/cwhub/dataset_test.go
View file

@ -6,6 +6,7 @@ import (
"github.com/jarcoal/httpmock" "github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestDownloadFile(t *testing.T) { func TestDownloadFile(t *testing.T) {
@ -14,12 +15,14 @@ func TestDownloadFile(t *testing.T) {
httpmock.Activate() httpmock.Activate()
defer httpmock.DeactivateAndReset() defer httpmock.DeactivateAndReset()
//OK //OK
httpmock.RegisterResponder( httpmock.RegisterResponder(
"GET", "GET",
"https://example.com/xx", "https://example.com/xx",
httpmock.NewStringResponder(200, "example content oneoneone"), httpmock.NewStringResponder(200, "example content oneoneone"),
) )
httpmock.RegisterResponder( httpmock.RegisterResponder(
"GET", "GET",
"https://example.com/x", "https://example.com/x",
@ -27,17 +30,21 @@ func TestDownloadFile(t *testing.T) {
) )
err := downloadFile("https://example.com/xx", examplePath) err := downloadFile("https://example.com/xx", examplePath)
assert.NoError(t, err) require.NoError(t, err)
content, err := os.ReadFile(examplePath) content, err := os.ReadFile(examplePath)
assert.Equal(t, "example content oneoneone", string(content)) assert.Equal(t, "example content oneoneone", string(content))
assert.NoError(t, err) require.NoError(t, err)
//bad uri //bad uri
err = downloadFile("https://zz.com", examplePath) err = downloadFile("https://zz.com", examplePath)
assert.Error(t, err) require.Error(t, err)
//404 //404
err = downloadFile("https://example.com/x", examplePath) err = downloadFile("https://example.com/x", examplePath)
assert.Error(t, err) require.Error(t, err)
//bad target //bad target
err = downloadFile("https://example.com/xx", "") err = downloadFile("https://example.com/xx", "")
assert.Error(t, err) require.Error(t, err)
} }

113
pkg/cwhub/doc.go Normal file
View file

@ -0,0 +1,113 @@
// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec.
//
// # Definitions
//
// - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items.
// - The hub INDEX is a JSON file that contains a tree of available hub items.
// - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version.
// - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items.
//
// Once downloaded, hub items can be installed by linking to them from the configuration directory.
// If an item is present in the configuration directory but it's not a link to the local hub, it is
// considered as a LOCAL ITEM and won't be removed or upgraded.
//
// # Directory Structure
//
// A typical directory layout is the following:
//
// For the local hub (HubDir = /etc/crowdsec/hub):
//
// - /etc/crowdsec/hub/.index.json
// - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
// - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
//
// For the configuration directory (InstallDir = /etc/crowdsec):
//
// - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
// - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
// - /etc/crowdsec/scenarios/local-scenario.yaml
//
// Note that installed items are not grouped by author, this may change in the future if we want to
// support items with the same name from different authors.
//
// Only parsers and postoverflows have the concept of stage.
//
// Additionally, an item can reference a DATA SET that is installed in a different location than
// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data).
//
// - /var/lib/crowdsec/data/http_path_traversal.txt
// - /var/lib/crowdsec/data/jira_cve_2021-26086.txt
// - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt
// - /var/lib/crowdsec/data/sensitive_data.txt
//
//
// # Using the package
//
// The main entry point is the Hub struct. You can create a new instance with NewHub().
// This constructor takes three parameters, but only the LOCAL HUB configuration is required:
//
// import (
// "fmt"
// "github.com/crowdsecurity/crowdsec/pkg/csconfig"
// "github.com/crowdsecurity/crowdsec/pkg/cwhub"
// )
//
// localHub := csconfig.LocalHubCfg{
// HubIndexFile: "/etc/crowdsec/hub/.index.json",
// HubDir: "/etc/crowdsec/hub",
// InstallDir: "/etc/crowdsec",
// InstallDataDir: "/var/lib/crowdsec/data",
// }
// hub, err := cwhub.NewHub(localHub, nil, false)
// if err != nil {
// return fmt.Errorf("unable to initialize hub: %w", err)
// }
//
// Now you can use the hub to access the existing items:
//
// // list all the parsers
// for _, parser := range hub.GetItemMap(cwhub.PARSERS) {
// fmt.Printf("parser: %s\n", parser.Name)
// }
//
// // retrieve a specific collection
// coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux")
// if coll == nil {
// return fmt.Errorf("collection not found")
// }
//
// You can also install items if they have already been downloaded:
//
// // install a parser
// force := false
// downloadOnly := false
// err := parser.Install(force, downloadOnly)
// if err != nil {
// return fmt.Errorf("unable to install parser: %w", err)
// }
//
// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash
// does not correspond to the latest version available in the index), a download will be attempted and you'll
// get the error "remote hub configuration is not provided".
//
// To provide the remote hub configuration, use the second parameter of NewHub():
//
// remoteHub := cwhub.RemoteHubCfg{
// URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// Branch: "master",
// IndexPath: ".index.json",
// }
// updateIndex := false
// hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex)
// if err != nil {
// return fmt.Errorf("unable to initialize hub: %w", err)
// }
//
// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
// placeholders: the branch and the file path (it will be an index or an item).
//
// Setting the third parameter to true will download the latest version of the index, if available on the
// specified branch.
// There is no exported method to update the index once the hub struct is created.
//
package cwhub

324
pkg/cwhub/download.go
View file

@ -1,324 +0,0 @@
package cwhub
import (
"bytes"
"crypto/sha256"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
var ErrIndexNotFound = fmt.Errorf("index not found")
func UpdateHubIdx(hub *csconfig.Hub) error {
bidx, err := DownloadHubIdx(hub)
if err != nil {
return fmt.Errorf("failed to download index: %w", err)
}
ret, err := LoadPkgIndex(bidx)
if err != nil {
if !errors.Is(err, ErrMissingReference) {
return fmt.Errorf("failed to read index: %w", err)
}
}
hubIdx = ret
if _, err := LocalSync(hub); err != nil {
return fmt.Errorf("failed to sync: %w", err)
}
return nil
}
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
if err != nil {
return nil, fmt.Errorf("failed to build request for hub index: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed http request for hub index: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
return nil, ErrIndexNotFound
}
return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
}
oldContent, err := os.ReadFile(hub.HubIndexFile)
if err != nil {
if !os.IsNotExist(err) {
log.Warningf("failed to read hub index: %s", err)
}
} else if bytes.Equal(body, oldContent) {
log.Info("hub index is up to date")
// write it anyway, can't hurt
}
file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, fmt.Errorf("while opening hub index file: %w", err)
}
defer file.Close()
wsize, err := file.Write(body)
if err != nil {
return nil, fmt.Errorf("while writing hub index file: %w", err)
}
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
return body, nil
}
// DownloadLatest will download the latest version of Item to the tdir directory
func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error {
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
if target.Type != COLLECTIONS {
if !target.Installed && updateOnly && target.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name)
return nil
}
return DownloadItem(hub, target, overwrite)
}
// collection
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
for idx, ptr := range tmp {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx[ptrtype][p]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
if !val.Installed && updateOnly && val.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name)
continue
}
log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
//recurse as it's a collection
if ptrtype == COLLECTIONS {
log.Tracef("collection, recurse")
err = DownloadLatest(hub, &val, overwrite, updateOnly)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
}
downloaded := val.Downloaded
err = DownloadItem(hub, &val, overwrite)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !val.Installed && !downloaded {
if err = EnableItem(hub, &val); err != nil {
return fmt.Errorf("enabling '%s': %w", val.Name, err)
}
}
hubIdx[ptrtype][p] = val
}
}
err = DownloadItem(hub, target, overwrite)
if err != nil {
return fmt.Errorf("failed to download item: %w", err)
}
return nil
}
func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
tdir := hub.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
if target.Tainted {
log.Debugf("%s : tainted, not updated", target.Name)
return nil
}
if target.UpToDate {
// We still have to check if data files are present
log.Debugf("%s : up-to-date, not updated", target.Name)
}
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
if err != nil {
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
}
h := sha256.New()
if _, err = h.Write(body); err != nil {
return fmt.Errorf("while hashing %s: %w", target.Name, err)
}
meow := fmt.Sprintf("%x", h.Sum(nil))
if meow != target.Versions[target.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
return fmt.Errorf("invalid download hash for %s", target.Name)
}
//all good, install
//check if parent dir exists
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
// ensure that target file is within target dir
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
}
if !strings.HasPrefix(finalPath, tdir) {
return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
}
// check dir
if _, err = os.Stat(parentDir); os.IsNotExist(err) {
log.Debugf("%s doesn't exist, create", parentDir)
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating parent directories: %w", err)
}
}
// check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s : overwrite", target.Name)
log.Debugf("target: %s/%s", tdir, target.RemotePath)
} else {
log.Infof("%s : OK", target.Name)
}
f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("while opening file: %w", err)
}
defer f.Close()
_, err = f.Write(body)
if err != nil {
return fmt.Errorf("while writing file: %w", err)
}
target.Downloaded = true
target.Tainted = false
target.UpToDate = true
if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
}
hubIdx[target.Type][target.Name] = *target
return nil
}
func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
itemFile, err := os.Open(itemFilePath)
if err != nil {
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
}
defer itemFile.Close()
if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}
return nil
}
func downloadData(dataFolder string, force bool, reader io.Reader) error {
var err error
dec := yaml.NewDecoder(reader)
for {
data := &DataSet{}
err = dec.Decode(data)
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("while reading file: %w", err)
}
download := false
for _, dataS := range data.Data {
if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
download = true
}
}
if download || force {
err = GetData(data.Data, dataFolder)
if err != nil {
return fmt.Errorf("while getting data: %w", err)
}
}
}
return nil
}

52
pkg/cwhub/download_test.go
View file

@ -1,52 +0,0 @@
package cwhub
import (
"fmt"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func TestDownloadHubIdx(t *testing.T) {
back := RawFileURLTemplate
// bad url template
fmt.Println("Test 'bad URL'")
RawFileURLTemplate = "x"
ret, err := DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
log.Errorf("unexpected error %s", err)
}
fmt.Printf("->%+v", ret)
// bad domain
fmt.Println("Test 'bad domain'")
RawFileURLTemplate = "https://baddomain/%s/%s"
ret, err = DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
log.Errorf("unexpected error %s", err)
}
fmt.Printf("->%+v", ret)
// bad target path
fmt.Println("Test 'bad target path'")
RawFileURLTemplate = back
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
log.Errorf("unexpected error %s", err)
}
RawFileURLTemplate = back
fmt.Printf("->%+v", ret)
}

190
pkg/cwhub/enable.go Normal file
View file

@ -0,0 +1,190 @@
package cwhub
// Enable/disable items already downloaded
import (
"fmt"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
)
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
// (eg. /etc/crowdsec/collections/xyz.yaml).
// Raises an error if the path goes outside of the install dir.
func (i *Item) installPath() (string, error) {
p := i.Type
if i.Stage != "" {
p = filepath.Join(p, i.Stage)
}
return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
}
// downloadPath returns the location of the actual config file in the hub
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
// Raises an error if the path goes outside of the hub dir.
func (i *Item) downloadPath() (string, error) {
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
if err != nil {
return "", err
}
return ret, nil
}
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
func (i *Item) createInstallLink() error {
dest, err := i.installPath()
if err != nil {
return err
}
destDir := filepath.Dir(dest)
if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating %s: %w", destDir, err)
}
if _, err = os.Lstat(dest); !os.IsNotExist(err) {
log.Infof("%s already exists.", dest)
return nil
}
src, err := i.downloadPath()
if err != nil {
return err
}
if err = os.Symlink(src, dest); err != nil {
return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
}
return nil
}
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
func (i *Item) enable() error {
if i.State.Installed {
if i.State.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
}
if i.IsLocal() {
return fmt.Errorf("%s is local, won't enable", i.Name)
}
// if it's a collection, check sub-items even if the collection file itself is up-to-date
if i.State.UpToDate && !i.HasSubItems() {
log.Tracef("%s is installed and up-to-date, skip.", i.Name)
return nil
}
}
for _, sub := range i.SubItems() {
if err := sub.enable(); err != nil {
return fmt.Errorf("while installing %s: %w", sub.Name, err)
}
}
if err := i.createInstallLink(); err != nil {
return err
}
log.Infof("Enabled %s: %s", i.Type, i.Name)
i.State.Installed = true
return nil
}
// purge removes the actual config file that was downloaded.
func (i *Item) purge() error {
if !i.State.Downloaded {
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
return nil
}
src, err := i.downloadPath()
if err != nil {
return err
}
if err := os.Remove(src); err != nil {
if os.IsNotExist(err) {
log.Debugf("%s doesn't exist, no need to remove", src)
return nil
}
return fmt.Errorf("while removing file: %w", err)
}
i.State.Downloaded = false
log.Infof("Removed source file [%s]: %s", i.Name, src)
return nil
}
// removeInstallLink removes the symlink to the downloaded content.
func (i *Item) removeInstallLink() error {
syml, err := i.installPath()
if err != nil {
return err
}
stat, err := os.Lstat(syml)
if err != nil {
return err
}
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
hubpath, err := os.Readlink(syml)
if err != nil {
return fmt.Errorf("while reading symlink: %w", err)
}
src, err := i.downloadPath()
if err != nil {
return err
}
if hubpath != src {
log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
if err := os.Remove(syml); err != nil {
return fmt.Errorf("while removing symlink: %w", err)
}
log.Infof("Removed symlink [%s]: %s", i.Name, syml)
return nil
}
// disable removes the install link, and optionally the downloaded content.
func (i *Item) disable(purge bool, force bool) error {
err := i.removeInstallLink()
if os.IsNotExist(err) {
if !purge && !force {
link, _ := i.installPath()
return fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
}
} else if err != nil {
return err
}
i.State.Installed = false
if purge {
if err := i.purge(); err != nil {
return err
}
}
return nil
}

141
pkg/cwhub/enable_test.go Normal file
View file

@ -0,0 +1,141 @@
package cwhub
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testInstall(hub *Hub, t *testing.T, item *Item) {
// Install the parser
_, err := item.downloadLatest(false, false)
require.NoError(t, err, "failed to download %s", item.Name)
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
err = item.enable()
require.NoError(t, err, "failed to enable %s", item.Name)
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
}
func testTaint(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
// truncate the file
f, err := os.Create(item.State.LocalPath)
require.NoError(t, err)
f.Close()
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name)
}
func testUpdate(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
_, err := item.downloadLatest(true, true)
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisable(hub *Hub, t *testing.T, item *Item) {
assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
// Remove
err := item.disable(false, false)
require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name)
// Purge
err = item.disable(true, false)
require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name)
}
func TestInstallParser(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
hub := envSetup(t)
// map iteration is random by itself
for _, it := range hub.Items[PARSERS] {
testInstall(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testTaint(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testUpdate(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testDisable(hub, t, it)
break
}
}
func TestInstallCollection(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
hub := envSetup(t)
// map iteration is random by itself
for _, it := range hub.Items[COLLECTIONS] {
testInstall(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testTaint(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testUpdate(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testDisable(hub, t, it)
break
}
}

21
pkg/cwhub/errors.go Normal file
View file

@ -0,0 +1,21 @@
package cwhub
import (
"errors"
"fmt"
)
var (
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
)
// IndexNotFoundError is returned when the remote hub index is not found.
type IndexNotFoundError struct {
URL string
Branch string
}
func (e IndexNotFoundError) Error() string {
return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch)
}

455
pkg/cwhub/helpers.go
View file

@ -1,222 +1,373 @@
package cwhub package cwhub
// Install, upgrade and remove items from the hub to the local configuration
import ( import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt" "fmt"
"io"
"net/http"
"os"
"path/filepath" "path/filepath"
"github.com/enescakir/emoji" "github.com/enescakir/emoji"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver" "slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
) )
// pick a hub branch corresponding to the current crowdsec version. // Install installs the item from the hub, downloading it if needed.
func chooseHubBranch() string { func (i *Item) Install(force bool, downloadOnly bool) error {
latest, err := cwversion.Latest() if downloadOnly && i.State.Downloaded && i.State.UpToDate {
if err != nil { log.Infof("%s is already downloaded and up-to-date", i.Name)
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
//lint:ignore nilerr
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("current version is equal to latest (%s)", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using master branch for the hub")
return "master"
}
log.Warnf("Crowdsec is not the latest version. "+
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
csVersion, latest)
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
"added to Crowdsec Hub after CrowdSec %s", latest)
return csVersion
}
// SetHubBranch sets the package variable that points to the hub branch.
func SetHubBranch() {
// a branch is already set, or specified from the flags
if HubBranch != "" {
return
}
// use the branch corresponding to the crowdsec version
HubBranch = chooseHubBranch()
log.Debugf("Using branch '%s' for the hub", HubBranch)
}
func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
item := GetItem(obtype, name)
if item == nil {
return fmt.Errorf("unable to retrieve item: %s", name)
}
if downloadOnly && item.Downloaded && item.UpToDate {
log.Warningf("%s is already downloaded and up-to-date", item.Name)
if !force { if !force {
return nil return nil
} }
} }
err := DownloadLatest(csConfig.Hub, item, force, true) filePath, err := i.downloadLatest(force, true)
if err != nil { if err != nil {
return fmt.Errorf("while downloading %s: %w", item.Name, err) return fmt.Errorf("while downloading %s: %w", i.Name, err)
}
if err = AddItem(obtype, *item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
} }
if downloadOnly { if downloadOnly {
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath)) log.Infof("Downloaded %s to %s", i.Name, filePath)
return nil return nil
} }
err = EnableItem(csConfig.Hub, item) if err := i.enable(); err != nil {
if err != nil { return fmt.Errorf("while enabling %s: %w", i.Name, err)
return fmt.Errorf("while enabling %s: %w", item.Name, err)
} }
if err := AddItem(obtype, *item); err != nil { log.Infof("Enabled %s", i.Name)
return fmt.Errorf("while adding %s: %w", item.Name, err)
}
log.Infof("Enabled %s", item.Name)
return nil return nil
} }
// XXX this must return errors instead of log.Fatal // descendants returns a list of all (direct or indirect) dependencies of the item.
func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) { func (i *Item) descendants() ([]*Item, error) {
if name != "" { var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
item := GetItem(itemType, name)
collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
if item == nil { if item == nil {
log.Fatalf("unable to retrieve: %s", name) return nil
} }
err := DisableItem(csConfig.Hub, item, purge, forceAction) if visited[item] {
return nil
}
visited[item] = true
for _, subItem := range item.SubItems() {
if subItem == i {
return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
}
*result = append(*result, subItem)
err := collectSubItems(subItem, visited, result)
if err != nil { if err != nil {
log.Fatalf("unable to disable %s : %v", item.Name, err) return err
}
} }
if err = AddItem(itemType, *item); err != nil { return nil
log.Fatalf("unable to add %s: %v", item.Name, err)
} }
return ret := []*Item{}
} visited := map[*Item]bool{}
if !all { err := collectSubItems(i, visited, &ret)
log.Fatal("removing item: no item specified")
}
disabled := 0
// remove all
for _, v := range GetItemMap(itemType) {
if !v.Installed {
continue
}
err := DisableItem(csConfig.Hub, &v, purge, forceAction)
if err != nil { if err != nil {
log.Fatalf("unable to disable %s : %v", v.Name, err) return nil, err
} }
if err := AddItem(itemType, v); err != nil { return ret, nil
log.Fatalf("unable to add %s: %v", v.Name, err)
}
disabled++
}
log.Infof("Disabled %d items", disabled)
} }
func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) { // Remove disables the item, optionally removing the downloaded content.
updated := 0 func (i *Item) Remove(purge bool, force bool) (bool, error) {
found := false if i.IsLocal() {
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
}
for _, v := range GetItemMap(itemType) { if i.State.Tainted && !force {
if name != "" && name != v.Name { return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
}
if !i.State.Installed && !purge {
log.Infof("removing %s: not installed -- no need to remove", i.Name)
return false, nil
}
removed := false
descendants, err := i.descendants()
if err != nil {
return false, err
}
ancestors := i.Ancestors()
for _, sub := range i.SubItems() {
if !sub.State.Installed {
continue continue
} }
if !v.Installed { // if the sub depends on a collection that is not a direct or indirect dependency
log.Tracef("skip %s, not installed", v.Name) // of the current item, it is not removed
for _, subParent := range sub.Ancestors() {
if !purge && !subParent.State.Installed {
continue continue
} }
if !v.Downloaded { // the ancestor that would block the removal of the sub item is also an ancestor
log.Warningf("%s : not downloaded, please install.", v.Name) // of the item we are removing, so we don't want false warnings
// (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux,
// while we are removing crowdsecurity/sshd)
if slices.Contains(ancestors, subParent) {
continue continue
} }
found = true // the sub-item belongs to the item we are removing, but we already knew that
if subParent == i {
continue
}
if v.UpToDate { if !slices.Contains(descendants, subParent) {
log.Infof("%s : up-to-date", v.Name) log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
continue
}
}
if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil { subRemoved, err := sub.Remove(purge, force)
log.Fatalf("%s : download failed : %v", v.Name, err) if err != nil {
return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
}
removed = removed || subRemoved
}
if err = i.disable(purge, force); err != nil {
return false, fmt.Errorf("while removing %s: %w", i.Name, err)
}
removed = true
return removed, nil
}
// Upgrade downloads and applies the last version of the item from the hub.
func (i *Item) Upgrade(force bool) (bool, error) {
updated := false
if !i.State.Downloaded {
return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
}
if !i.State.Installed {
return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name)
}
if i.State.UpToDate {
log.Infof("%s: up-to-date", i.Name)
if err := i.DownloadDataIfNeeded(force); err != nil {
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
} }
if !force { if !force {
// no upgrade needed
return false, nil
}
}
if _, err := i.downloadLatest(force, true); err != nil {
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
}
if !i.State.UpToDate {
if i.State.Tainted {
log.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
} else if i.IsLocal() {
log.Infof("%v %s is local", emoji.Prohibited, i.Name)
}
} else {
// a check on stdout is used while scripting to know if the hub has been upgraded
// and a configuration reload is required
// TODO: use a better way to communicate this
fmt.Printf("updated %s\n", i.Name)
log.Infof("%v %s: updated", emoji.Package, i.Name)
updated = true
}
return updated, nil
}
// downloadLatest downloads the latest version of the item to the hub directory.
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
log.Debugf("Downloading %s %s", i.Type, i.Name)
for _, sub := range i.SubItems() {
if !sub.State.Installed && updateOnly && sub.State.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name)
continue continue
} }
}
if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil { log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly)
log.Fatalf("%s : download failed : %v", v.Name, err)
}
if !v.UpToDate { // recurse as it's a collection
if v.Tainted { if sub.HasSubItems() {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name) log.Tracef("collection, recurse")
} else if v.Local {
log.Infof("%v %s is local", emoji.Prohibited, v.Name)
}
} else {
// this is used while scripting to know if the hub has been upgraded
// and a configuration reload is required
fmt.Printf("updated %s\n", v.Name)
log.Infof("%v %s : updated", emoji.Package, v.Name)
updated++
}
if err := AddItem(itemType, v); err != nil { if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil {
log.Fatalf("unable to add %s: %v", v.Name, err) return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
} }
} }
if !found && name == "" { downloaded := sub.State.Downloaded
log.Infof("No %s installed, nothing to upgrade", itemType)
} else if !found { if _, err := sub.download(overwrite); err != nil {
log.Errorf("Item '%s' not found in hub", name) return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
} else if updated == 0 && found {
if name == "" {
log.Infof("All %s are already up-to-date", itemType)
} else {
log.Infof("Item '%s' is up-to-date", name)
} }
} else if updated != 0 {
log.Infof("Upgraded %d items", updated) // We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if sub.Downloaded is false because maybe the item has been disabled by the user.
if !sub.State.Installed && !downloaded {
if err := sub.enable(); err != nil {
return "", fmt.Errorf("enabling '%s': %w", sub.Name, err)
} }
}
}
if !i.State.Installed && updateOnly && i.State.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name)
return "", nil
}
ret, err := i.download(overwrite)
if err != nil {
return "", fmt.Errorf("failed to download item: %w", err)
}
return ret, nil
}
// fetch downloads the item from the hub, verifies the hash and returns the content.
func (i *Item) fetch() ([]byte, error) {
url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
return nil, fmt.Errorf("failed to build hub item request: %w", err)
}
resp, err := hubClient.Get(url)
if err != nil {
return nil, fmt.Errorf("while downloading %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("while downloading %s: %w", url, err)
}
hash := sha256.New()
if _, err = hash.Write(body); err != nil {
return nil, fmt.Errorf("while hashing %s: %w", i.Name, err)
}
meow := hex.EncodeToString(hash.Sum(nil))
if meow != i.Versions[i.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
return nil, fmt.Errorf("invalid download hash for %s", i.Name)
}
return body, nil
}
// download downloads the item from the hub and writes it to the hub directory.
func (i *Item) download(overwrite bool) (string, error) {
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
if i.State.Tainted {
log.Debugf("%s: tainted, not updated", i.Name)
return "", nil
}
if i.State.UpToDate {
// We still have to check if data files are present
log.Debugf("%s: up-to-date, not updated", i.Name)
}
}
body, err := i.fetch()
if err != nil {
return "", err
}
// all good, install
// ensure that target file is within target dir
finalPath, err := i.downloadPath()
if err != nil {
return "", err
}
parentDir := filepath.Dir(finalPath)
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
return "", fmt.Errorf("while creating %s: %w", parentDir, err)
}
// check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s: overwrite", i.Name)
log.Debugf("target: %s", finalPath)
} else {
log.Infof("%s: OK", i.Name)
}
if err = os.WriteFile(finalPath, body, 0o644); err != nil {
return "", fmt.Errorf("while writing %s: %w", finalPath, err)
}
i.State.Downloaded = true
i.State.Tainted = false
i.State.UpToDate = true
if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
}
return finalPath, nil
}
// DownloadDataIfNeeded downloads the data set for the item.
func (i *Item) DownloadDataIfNeeded(force bool) error {
itemFilePath, err := i.installPath()
if err != nil {
return err
}
itemFile, err := os.Open(itemFilePath)
if err != nil {
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
}
defer itemFile.Close()
if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}
return nil
} }

218
pkg/cwhub/helpers_test.go
View file

@ -4,157 +4,187 @@ import (
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
) )
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection.
// We expect the new scenario to be installed // We expect the new scenario to be installed.
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
cfg := envSetup(t) hub := envSetup(t)
defer envTearDown(cfg)
// fresh install of collection // fresh install of collection
getHubIdxOrFail(t) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.NoError(t, item.Install(false, false))
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
// This is the scenario that gets added in next version of collection // This is the scenario that gets added in next version of collection
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
pushUpdateToCollectionInHub() pushUpdateToCollectionInHub()
if err := UpdateHubIdx(cfg.Hub); err != nil { remote := &RemoteHubCfg{
t.Fatalf("failed to download index : %s", err) URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
} }
getHubIdxOrFail(t) hub, err := NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) hub = getHubOrFail(t, hub.local, remote)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.True(t, didUpdate)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
} }
// Install a collection, disable a scenario. // Install a collection, disable a scenario.
// Upgrade should install should not enable/download the disabled scenario. // Upgrade should install should not enable/download the disabled scenario.
func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
cfg := envSetup(t) hub := envSetup(t)
defer envTearDown(cfg)
// fresh install of collection // fresh install of collection
getHubIdxOrFail(t) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.NoError(t, item.Install(false, false))
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) didRemove, err := item.Remove(false, false)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.NoError(t, err)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) require.True(t, didRemove)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) remote := &RemoteHubCfg{
getHubIdxOrFail(t) URLTemplate: mockURLTemplate,
// scenario referenced by collection was deleted hence, collection should be tainted Branch: "master",
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) IndexPath: ".index.json",
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
if err := UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err)
} }
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
getHubIdxOrFail(t) hub, err = NewHub(hub.local, remote, true)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.NoError(t, err, "failed to download index: %s", err)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.False(t, didUpdate)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
} }
func getHubIdxOrFail(t *testing.T) { // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
if err := GetHubIdx(getTestCfg().Hub); err != nil { func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
t.Fatalf("failed to load hub index") hub, err := NewHub(local, remote, false)
} require.NoError(t, err, "failed to load hub index")
return hub
} }
// Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario // Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
// Upgrade should not enable/download the disabled scenario. // Upgrade should not enable/download the disabled scenario.
// Upgrade should install and enable the newly added scenario. // Upgrade should install and enable the newly added scenario.
func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
cfg := envSetup(t) hub := envSetup(t)
defer envTearDown(cfg)
// fresh install of collection // fresh install of collection
getHubIdxOrFail(t) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.NoError(t, item.Install(false, false))
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) didRemove, err := item.Remove(false, false)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.NoError(t, err)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) require.True(t, didRemove)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) remote := &RemoteHubCfg{
getHubIdxOrFail(t) URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted // scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
// we just removed. Nor should it install the newly added scenario // we just removed. Nor should it install the newly added scenario
pushUpdateToCollectionInHub() pushUpdateToCollectionInHub()
if err := UpdateHubIdx(cfg.Hub); err != nil { hub, err = NewHub(hub.local, remote, true)
t.Fatalf("failed to download index : %s", err) require.NoError(t, err, "failed to download index: %s", err)
}
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
getHubIdxOrFail(t) hub = getHubOrFail(t, hub.local, remote)
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
getHubIdxOrFail(t) didUpdate, err := item.Upgrade(false)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) require.NoError(t, err)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) require.True(t, didUpdate)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
} }
func assertCollectionDepsInstalled(t *testing.T, collection string) { func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
t.Helper() t.Helper()
c := hubIdx[COLLECTIONS][collection] c := hub.Items[COLLECTIONS][collection]
require.NoError(t, CollecDepsCheck(&c)) require.NoError(t, c.checkSubItemVersions())
} }
func pushUpdateToCollectionInHub() { func pushUpdateToCollectionInHub() {

161
pkg/cwhub/hub.go Normal file
View file

@ -0,0 +1,161 @@
package cwhub
import (
"bytes"
"encoding/json"
"fmt"
"os"
"path"
"strings"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
// Hub is the main structure for the package.
type Hub struct {
Items HubItems // Items read from HubDir and InstallDir
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
Warnings []string // Warnings encountered during sync
}
// GetDataDir returns the data directory, where data sets are installed.
func (h *Hub) GetDataDir() string {
return h.local.InstallDataDir
}
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
// If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
if local == nil {
return nil, fmt.Errorf("no hub configuration found")
}
hub := &Hub{
local: local,
remote: remote,
}
if updateIndex {
if err := hub.updateIndex(); err != nil {
return nil, err
}
}
log.Debugf("loading hub idx %s", local.HubIndexFile)
if err := hub.parseIndex(); err != nil {
return nil, fmt.Errorf("failed to load index: %w", err)
}
if err := hub.localSync(); err != nil {
return nil, fmt.Errorf("failed to sync items: %w", err)
}
return hub, nil
}
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections.
func (h *Hub) parseIndex() error {
bidx, err := os.ReadFile(h.local.HubIndexFile)
if err != nil {
return fmt.Errorf("unable to read index file: %w", err)
}
if err := json.Unmarshal(bidx, &h.Items); err != nil {
return fmt.Errorf("failed to unmarshal index: %w", err)
}
log.Debugf("%d item types in hub index", len(ItemTypes))
// Iterate over the different types to complete the struct
for _, itemType := range ItemTypes {
log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
for name, item := range h.Items[itemType] {
item.hub = h
item.Name = name
// if the item has no (redundant) author, take it from the json key
if item.Author == "" && strings.Contains(name, "/") {
item.Author = strings.Split(name, "/")[0]
}
item.Type = itemType
item.FileName = path.Base(item.RemotePath)
item.logMissingSubItems()
}
}
return nil
}
// ItemStats returns total counts of the hub items, including local and tainted.
func (h *Hub) ItemStats() []string {
loaded := ""
local := 0
tainted := 0
for _, itemType := range ItemTypes {
if len(h.Items[itemType]) == 0 {
continue
}
loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
for _, item := range h.Items[itemType] {
if item.IsLocal() {
local++
}
if item.State.Tainted {
tainted++
}
}
}
loaded = strings.Trim(loaded, ", ")
if loaded == "" {
loaded = "0 items"
}
ret := []string{
fmt.Sprintf("Loaded: %s", loaded),
}
if local > 0 || tainted > 0 {
ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", local, tainted))
}
return ret
}
// updateIndex downloads the latest version of the index and writes it to disk if it changed.
func (h *Hub) updateIndex() error {
body, err := h.remote.fetchIndex()
if err != nil {
return err
}
oldContent, err := os.ReadFile(h.local.HubIndexFile)
if err != nil {
if !os.IsNotExist(err) {
log.Warningf("failed to read hub index: %s", err)
}
} else if bytes.Equal(body, oldContent) {
log.Info("hub index is up to date")
return nil
}
if err = os.WriteFile(h.local.HubIndexFile, body, 0o644); err != nil {
return fmt.Errorf("failed to write hub index: %w", err)
}
log.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body))
return nil
}

77
pkg/cwhub/hub_test.go Normal file
View file

@ -0,0 +1,77 @@
package cwhub
import (
"fmt"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestInitHubUpdate(t *testing.T) {
hub := envSetup(t)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
_, err := NewHub(hub.local, remote, true)
require.NoError(t, err)
}
func TestUpdateIndex(t *testing.T) {
// bad url template
fmt.Println("Test 'bad URL'")
tmpIndex, err := os.CreateTemp("", "index.json")
require.NoError(t, err)
t.Cleanup(func() {
os.Remove(tmpIndex.Name())
})
hub := envSetup(t)
hub.remote = &RemoteHubCfg{
URLTemplate: "x",
Branch: "",
IndexPath: "",
}
hub.local.HubIndexFile = tmpIndex.Name()
err = hub.updateIndex()
cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'")
// bad domain
fmt.Println("Test 'bad domain'")
hub.remote = &RemoteHubCfg{
URLTemplate: "https://baddomain/%s/%s",
Branch: "master",
IndexPath: ".index.json",
}
err = hub.updateIndex()
require.NoError(t, err)
// XXX: this is not failing
// cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
// bad target path
fmt.Println("Test 'bad target path'")
hub.remote = &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
hub.local.HubIndexFile = "/does/not/exist/index.json"
err = hub.updateIndex()
cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:")
}

214
pkg/cwhub/install.go
View file

@ -1,214 +0,0 @@
package cwhub
import (
"fmt"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
itempath := hub.HubDir + "/" + target.RemotePath
// disable hub file
if err := os.Remove(itempath); err != nil {
return target, fmt.Errorf("while removing file: %w", err)
}
target.Downloaded = false
log.Infof("Removed source file [%s]: %s", target.Name, itempath)
hubIdx[target.Type][target.Name] = target
return target, nil
}
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error {
var err error
// already disabled, noop unless purge
if !target.Installed {
if purge {
*target, err = purgeItem(hub, *target)
if err != nil {
return err
}
}
return nil
}
if target.Local {
return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
}
if target.Tainted && !force {
return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
}
// for a COLLECTIONS, disable sub-items
if target.Type == COLLECTIONS {
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if val, ok := hubIdx[ptrtype][p]; ok {
// check if the item doesn't belong to another collection before removing it
toRemove := true
for _, collection := range val.BelongsToCollections {
if collection != target.Name {
toRemove = false
break
}
}
if toRemove {
err = DisableItem(hub, &val, purge, force)
if err != nil {
return fmt.Errorf("while disabling %s: %w", p, err)
}
} else {
log.Infof("%s was not removed because it belongs to another collection", val.Name)
}
} else {
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
}
}
}
}
syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
if err != nil {
return err
}
stat, err := os.Lstat(syml)
if os.IsNotExist(err) {
// we only accept to "delete" non existing items if it's a forced purge
if !purge && !force {
return fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml)
}
} else {
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml)
return fmt.Errorf("%s isn't managed by hub", target.Name)
}
hubpath, err := os.Readlink(syml)
if err != nil {
return fmt.Errorf("while reading symlink: %w", err)
}
absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while abs path: %w", err)
}
if hubpath != absPath {
log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath)
return fmt.Errorf("%s isn't managed by hub", target.Name)
}
// remove the symlink
if err = os.Remove(syml); err != nil {
return fmt.Errorf("while removing symlink: %w", err)
}
log.Infof("Removed symlink [%s] : %s", target.Name, syml)
}
target.Installed = false
if purge {
*target, err = purgeItem(hub, *target)
if err != nil {
return err
}
}
hubIdx[target.Type][target.Name] = *target
return nil
}
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
// Handles collections recursively
func EnableItem(hub *csconfig.Hub, target *Item) error {
var err error
parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
// create directories if needed
if target.Installed {
if target.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
}
if target.Local {
return fmt.Errorf("%s is local, won't enable", target.Name)
}
// if it's a collection, check sub-items even if the collection file itself is up-to-date
if target.UpToDate && target.Type != COLLECTIONS {
log.Tracef("%s is installed and up-to-date, skip.", target.Name)
return nil
}
}
if _, err = os.Stat(parentDir); os.IsNotExist(err) {
log.Infof("%s doesn't exist, create", parentDir)
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating directory: %w", err)
}
}
// install sub-items if it's a collection
if target.Type == COLLECTIONS {
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx[ptrtype][p]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
err = EnableItem(hub, &val)
if err != nil {
return fmt.Errorf("while installing %s: %w", p, err)
}
}
}
}
// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) {
log.Infof("%s already exists.", parentDir+"/"+target.FileName)
return nil
}
// hub.ConfigDir + target.RemotePath
srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while getting source path: %w", err)
}
dstPath, err := filepath.Abs(parentDir + "/" + target.FileName)
if err != nil {
return fmt.Errorf("while getting destination path: %w", err)
}
if err = os.Symlink(srcPath, dstPath); err != nil {
return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
}
log.Infof("Enabled %s : %s", target.Type, target.Name)
target.Installed = true
hubIdx[target.Type][target.Name] = *target
return nil
}

383
pkg/cwhub/items.go Normal file
View file

@ -0,0 +1,383 @@
package cwhub
import (
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/Masterminds/semver/v3"
"github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
)
const (
// managed item types.
COLLECTIONS = "collections"
PARSERS = "parsers"
POSTOVERFLOWS = "postoverflows"
SCENARIOS = "scenarios"
)
const (
versionUpToDate = iota // the latest version from index is installed
versionUpdateAvailable // not installed, or lower than latest
versionUnknown // local file with no version, or invalid version number
versionFuture // local version is higher latest, but is included in the index: should not happen
)
var (
// The order is important, as it is used to range over sub-items in collections.
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
)
type HubItems map[string]map[string]*Item
// ItemVersion is used to detect the version of a given item
// by comparing the hash of each version to the local file.
// If the item does not match any known version, it is considered tainted (modified).
type ItemVersion struct {
Digest string `json:"digest,omitempty" yaml:"digest,omitempty"`
Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"`
}
// ItemState is used to keep the local state (i.e. at runtime) of an item.
// This data is not stored in the index, but is displayed with "cscli ... inspect".
type ItemState struct {
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"`
LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"`
Installed bool `json:"installed"`
Downloaded bool `json:"downloaded"`
UpToDate bool `json:"up_to_date"`
Tainted bool `json:"tainted"`
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
}
// Item is created from an index file and enriched with local info.
type Item struct {
hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
State ItemState `json:"-" yaml:"-"` // local state, not stored in the index
Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
Name string `json:"name,omitempty" yaml:"name,omitempty"` // usually "author/name"
FileName string `json:"file_name,omitempty" yaml:"file_name,omitempty"` // eg. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"`
Author string `json:"author,omitempty" yaml:"author,omitempty"`
References []string `json:"references,omitempty" yaml:"references,omitempty"`
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty" yaml:"version,omitempty"` // the last available version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions
// if it's a collection, it can have sub items
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
func (i *Item) HasSubItems() bool {
return i.Type == COLLECTIONS
}
// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
func (i *Item) IsLocal() bool {
return i.State.Installed && !i.State.Downloaded
}
// MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
// It must not use a pointer receiver.
func (i Item) MarshalJSON() ([]byte, error) {
type Alias Item
return json.Marshal(&struct {
Alias
// we have to repeat the fields here, json will have inline support in v2
LocalPath string `json:"local_path,omitempty"`
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"`
Installed bool `json:"installed"`
Downloaded bool `json:"downloaded"`
UpToDate bool `json:"up_to_date"`
Tainted bool `json:"tainted"`
Local bool `json:"local"`
BelongsToCollections []string `json:"belongs_to_collections,omitempty"`
}{
Alias: Alias(i),
LocalPath: i.State.LocalPath,
LocalVersion: i.State.LocalVersion,
LocalHash: i.State.LocalHash,
Installed: i.State.Installed,
Downloaded: i.State.Downloaded,
UpToDate: i.State.UpToDate,
Tainted: i.State.Tainted,
BelongsToCollections: i.State.BelongsToCollections,
Local: i.IsLocal(),
})
}
// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw".
// It must not use a pointer receiver.
func (i Item) MarshalYAML() (interface{}, error) {
type Alias Item
return &struct {
Alias `yaml:",inline"`
State ItemState `yaml:",inline"`
Local bool `yaml:"local"`
}{
Alias: Alias(i),
State: i.State,
Local: i.IsLocal(),
}, nil
}
// SubItems returns a slice of sub-items, excluding the ones that were not found.
func (i *Item) SubItems() []*Item {
sub := make([]*Item, 0)
for _, name := range i.Parsers {
s := i.hub.GetItem(PARSERS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.PostOverflows {
s := i.hub.GetItem(POSTOVERFLOWS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.Scenarios {
s := i.hub.GetItem(SCENARIOS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.Collections {
s := i.hub.GetItem(COLLECTIONS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
return sub
}
func (i *Item) logMissingSubItems() {
if !i.HasSubItems() {
return
}
for _, subName := range i.Parsers {
if i.hub.GetItem(PARSERS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name)
}
}
for _, subName := range i.Scenarios {
if i.hub.GetItem(SCENARIOS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name)
}
}
for _, subName := range i.PostOverflows {
if i.hub.GetItem(POSTOVERFLOWS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name)
}
}
for _, subName := range i.Collections {
if i.hub.GetItem(COLLECTIONS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name)
}
}
}
// Ancestors returns a slice of items (typically collections) that have this item as a direct or indirect dependency.
func (i *Item) Ancestors() []*Item {
ret := make([]*Item, 0)
for _, parentName := range i.State.BelongsToCollections {
parent := i.hub.GetItem(COLLECTIONS, parentName)
if parent == nil {
continue
}
ret = append(ret, parent)
}
return ret
}
// InstallStatus returns the status of the item as a string and an emoji
// (eg. "enabled,update-available" and emoji.Warning).
func (i *Item) InstallStatus() (string, emoji.Emoji) {
status := "disabled"
ok := false
if i.State.Installed {
ok = true
status = "enabled"
}
managed := true
if i.IsLocal() {
managed = false
status += ",local"
}
warning := false
if i.State.Tainted {
warning = true
status += ",tainted"
} else if !i.State.UpToDate && !i.IsLocal() {
warning = true
status += ",update-available"
}
emo := emoji.QuestionMark
switch {
case !managed:
emo = emoji.House
case !i.State.Installed:
emo = emoji.Prohibited
case warning:
emo = emoji.Warning
case ok:
emo = emoji.CheckMark
}
return status, emo
}
// versionStatus returns the status of the item version compared to the hub version.
// semver requires the 'v' prefix.
func (i *Item) versionStatus() int {
local, err := semver.NewVersion(i.State.LocalVersion)
if err != nil {
return versionUnknown
}
// hub versions are already validated while syncing, ignore errors
latest, _ := semver.NewVersion(i.Version)
if local.LessThan(latest) {
return versionUpdateAvailable
}
if local.Equal(latest) {
return versionUpToDate
}
return versionFuture
}
// validPath returns true if the (relative) path is allowed for the item.
// dirNname: the directory name (ie. crowdsecurity).
// fileName: the filename (ie. apache2-logs.yaml).
func (i *Item) validPath(dirName, fileName string) bool {
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
}
// GetItemMap returns the map of items for a given type.
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
return h.Items[itemType]
}
// GetItem returns an item from hub based on its type and full name (author/name).
func (h *Hub) GetItem(itemType string, itemName string) *Item {
return h.GetItemMap(itemType)[itemName]
}
// GetItemNames returns a slice of (full) item names for a given type
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
func (h *Hub) GetItemNames(itemType string) []string {
m := h.GetItemMap(itemType)
if m == nil {
return nil
}
names := make([]string, 0, len(m))
for k := range m {
names = append(names, k)
}
return names
}
// GetAllItems returns a slice of all the items of a given type, installed or not.
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
ret := make([]*Item, len(items))
idx := 0
for _, item := range items {
ret[idx] = item
idx++
}
return ret, nil
}
// GetInstalledItems returns a slice of the installed items of a given type.
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
retItems := make([]*Item, 0)
for _, item := range items {
if item.State.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
// GetInstalledItemNames returns the names of the installed items of a given type.
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
items, err := h.GetInstalledItems(itemType)
if err != nil {
return nil, err
}
retStr := make([]string, len(items))
for idx, it := range items {
retStr[idx] = it.Name
}
return retStr, nil
}
// SortItemSlice sorts a slice of items by name, case insensitive.
func SortItemSlice(items []*Item) {
sort.Slice(items, func(i, j int) bool {
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
})
}

71
pkg/cwhub/items_test.go Normal file
View file

@ -0,0 +1,71 @@
package cwhub
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestItemStatus(t *testing.T) {
hub := envSetup(t)
// get existing map
x := hub.GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item: good and bad
for k := range x {
item := hub.GetItem(COLLECTIONS, k)
require.NotNil(t, item)
item.State.Installed = true
item.State.UpToDate = false
item.State.Tainted = false
item.State.Downloaded = true
txt, _ := item.InstallStatus()
require.Equal(t, "enabled,update-available", txt)
item.State.Installed = true
item.State.UpToDate = false
item.State.Tainted = false
item.State.Downloaded = false
txt, _ = item.InstallStatus()
require.Equal(t, "enabled,local", txt)
}
stats := hub.ItemStats()
require.Equal(t, []string{
"Loaded: 2 parsers, 1 scenarios, 3 collections",
"Unmanaged items: 3 local, 0 tainted",
}, stats)
}
func TestGetters(t *testing.T) {
hub := envSetup(t)
// get non existing map
empty := hub.GetItemMap("ratata")
require.Nil(t, empty)
// get existing map
x := hub.GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item: good and bad
for k := range x {
empty := hub.GetItem(COLLECTIONS, k+"nope")
require.Nil(t, empty)
item := hub.GetItem(COLLECTIONS, k)
require.NotNil(t, item)
// Add item and get it
item.Name += "nope"
hub.Items[item.Type][item.Name] = item
newitem := hub.GetItem(COLLECTIONS, item.Name)
require.NotNil(t, newitem)
}
}

53
pkg/cwhub/leakybucket.go Normal file
View file

@ -0,0 +1,53 @@
package cwhub
// Resolve a symlink to find the hub item it points to.
// This file is used only by pkg/leakybucket
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary.
func itemKey(itemPath string) (string, error) {
f, err := os.Lstat(itemPath)
if err != nil {
return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
}
if f.Mode()&os.ModeSymlink == 0 {
// it's not a symlink, so the filename itsef should be the key
return filepath.Base(itemPath), nil
}
// resolve the symlink to hub file
pathInHub, err := os.Readlink(itemPath)
if err != nil {
return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
}
author := filepath.Base(filepath.Dir(pathInHub))
fname := filepath.Base(pathInHub)
fname = strings.TrimSuffix(fname, ".yaml")
fname = strings.TrimSuffix(fname, ".yml")
return fmt.Sprintf("%s/%s", author, fname), nil
}
// GetItemByPath retrieves an item from the hub index based on its local path.
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
itemKey, err := itemKey(itemPath)
if err != nil {
return nil, err
}
item := h.GetItem(itemType, itemKey)
if item == nil {
return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
}
return item, nil
}

552
pkg/cwhub/loader.go
View file

@ -1,552 +0,0 @@
package cwhub
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func isYAMLFileName(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
func validItemFileName(vname string, fauthor string, fname string) bool {
return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml")
}
func handleSymlink(path string) (string, error) {
hubpath, err := os.Readlink(path)
if err != nil {
return "", fmt.Errorf("unable to read symlink of %s", path)
}
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
_, err = os.Lstat(hubpath)
if os.IsNotExist(err) {
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
// remove the symlink
if err = os.Remove(path); err != nil {
return "", fmt.Errorf("failed to unlink %s: %w", path, err)
}
// XXX: is this correct?
return "", nil
}
return hubpath, nil
}
func getSHA256(filepath string) (string, error) {
f, err := os.Open(filepath)
if err != nil {
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
type Walker struct {
// the walk/parserVisit function can't receive extra args
hubdir string
installdir string
}
func NewWalker(hub *csconfig.Hub) Walker {
return Walker{
hubdir: hub.HubDir,
installdir: hub.InstallDir,
}
}
type itemFileInfo struct {
fname string
stage string
ftype string
fauthor string
}
func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
ret := itemFileInfo{}
inhub := false
subs := strings.Split(path, string(os.PathSeparator))
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir)
log.Tracef("subs:%v", subs)
// we're in hub (~/.hub/hub/)
if strings.HasPrefix(path, w.hubdir) {
log.Tracef("in hub dir")
inhub = true
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml
//.../hub/profiles/crowdsec/linux.yaml
if len(subs) < 4 {
log.Fatalf("path is too short : %s (%d)", path, len(subs))
}
ret.fname = subs[len(subs)-1]
ret.fauthor = subs[len(subs)-2]
ret.stage = subs[len(subs)-3]
ret.ftype = subs[len(subs)-4]
} else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir")
if len(subs) < 3 {
log.Fatalf("path is too short : %s (%d)", path, len(subs))
}
///.../config/parser/stage/file.yaml
///.../config/postoverflow/stage/file.yaml
///.../config/scenarios/scenar.yaml
///.../config/collections/linux.yaml //file is empty
ret.fname = subs[len(subs)-1]
ret.stage = subs[len(subs)-2]
ret.ftype = subs[len(subs)-3]
ret.fauthor = ""
} else {
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir)
}
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
if ret.stage == SCENARIOS {
ret.ftype = SCENARIOS
ret.stage = ""
} else if ret.stage == COLLECTIONS {
ret.ftype = COLLECTIONS
ret.stage = ""
} else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW {
// its a PARSER / PARSER_OVFLW with a stage
return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
}
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
return ret, inhub, nil
}
func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
var (
local bool
hubpath string
)
if err != nil {
log.Debugf("while syncing hub dir: %s", err)
// there is a path error, we ignore the file
return nil
}
path, err = filepath.Abs(path)
if err != nil {
return err
}
// we only care about files
if f == nil || f.IsDir() {
return nil
}
if !isYAMLFileName(f.Name()) {
return nil
}
info, inhub, err := w.getItemInfo(path)
if err != nil {
return err
}
/*
we can encounter 'collections' in the form of a symlink :
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
when the collection is installed, both files are created
*/
// non symlinks are local user files or hub files
if f.Type()&os.ModeSymlink == 0 {
local = true
log.Tracef("%s isn't a symlink", path)
} else {
hubpath, err = handleSymlink(path)
if err != nil {
return err
}
log.Tracef("%s points to %s", path, hubpath)
if hubpath == "" {
// XXX: is this correct?
return nil
}
}
// if it's not a symlink and not in hub, it's a local file, don't bother
if local && !inhub {
log.Tracef("%s is a local file, skip", path)
skippedLocal++
// log.Infof("local scenario, skip.")
_, fileName := filepath.Split(path)
hubIdx[info.ftype][info.fname] = Item{
Name: info.fname,
Stage: info.stage,
Installed: true,
Type: info.ftype,
Local: true,
LocalPath: path,
UpToDate: true,
FileName: fileName,
}
return nil
}
// try to find which configuration item it is
log.Tracef("check [%s] of %s", info.fname, info.ftype)
match := false
for name, item := range hubIdx[info.ftype] {
log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
if info.fname != item.FileName {
log.Tracef("%s != %s (filename)", info.fname, item.FileName)
continue
}
// wrong stage
if item.Stage != info.stage {
continue
}
// if we are walking hub dir, just mark present files as downloaded
if inhub {
// wrong author
if info.fauthor != item.Author {
continue
}
// wrong file
if !validItemFileName(item.Name, info.fauthor, info.fname) {
continue
}
if path == w.hubdir+"/"+item.RemotePath {
log.Tracef("marking %s as downloaded", item.Name)
item.Downloaded = true
}
} else if !hasPathSuffix(hubpath, item.RemotePath) {
// wrong file
// <type>/<stage>/<author>/<name>.yaml
continue
}
sha, err := getSHA256(path)
if err != nil {
log.Fatalf("Failed to get sha of %s : %v", path, err)
}
// let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(item.Versions))
for k := range item.Versions {
versions = append(versions, k)
}
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
for _, version := range versions {
val := item.Versions[version]
if sha != val.Digest {
// log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
continue
}
// we got an exact match, update struct
item.Downloaded = true
item.LocalHash = sha
if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version)
item.LocalPath = path
item.LocalVersion = version
item.Tainted = false
// if we're walking the hub, present file doesn't means installed file
item.Installed = true
}
if version == item.Version {
log.Tracef("%s is up-to-date", item.Name)
item.UpToDate = true
}
match = true
break
}
if !match {
log.Tracef("got tainted match for %s: %s", item.Name, path)
skippedTainted++
// the file and the stage is right, but the hash is wrong, it has been tainted by user
if !inhub {
item.LocalPath = path
item.Installed = true
}
item.UpToDate = false
item.LocalVersion = "?"
item.Tainted = true
item.LocalHash = sha
}
// update the entry if appropriate
// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
// fmt.Printf("Updating %s", k)
// hubIdx[ftype][k] = v
// } else if !inhub {
// } else if
hubIdx[info.ftype][name] = item
return nil
}
log.Infof("Ignoring file %s of type %s", path, info.ftype)
return nil
}
func CollecDepsCheck(v *Item) error {
if v.versionStatus() != 0 { // not up-to-date
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
return nil
}
if v.Type != COLLECTIONS {
return nil
}
// if it's a collection, ensure all the items are installed, or tag it as tainted
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
sliceType := ItemTypes[idx]
for _, subName := range itemSlice {
subItem, ok := hubIdx[sliceType][subName]
if !ok {
log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name)
}
log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
if !v.Installed {
continue
}
if subItem.Type == COLLECTIONS {
log.Tracef("collec, recurse.")
if err := CollecDepsCheck(&subItem); err != nil {
if subItem.Tainted {
v.Tainted = true
}
return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
}
hubIdx[sliceType][subName] = subItem
}
// propagate the state of sub-items to set
if subItem.Tainted {
v.Tainted = true
return fmt.Errorf("tainted %s %s, tainted", sliceType, subName)
}
if !subItem.Installed && v.Installed {
v.Tainted = true
return fmt.Errorf("missing %s %s, tainted", sliceType, subName)
}
if !subItem.UpToDate {
v.UpToDate = false
return fmt.Errorf("outdated %s %s", sliceType, subName)
}
skip := false
for idx := range subItem.BelongsToCollections {
if subItem.BelongsToCollections[idx] == v.Name {
skip = true
}
}
if !skip {
subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
}
hubIdx[sliceType][subName] = subItem
log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
}
}
return nil
}
func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
warnings := []string{}
// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
for _, scan := range ItemTypes {
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
if err != nil {
log.Errorf("failed %s : %s", cpath, err)
}
err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit)
if err != nil {
return warnings, err
}
}
for name, item := range hubIdx[COLLECTIONS] {
if !item.Installed {
continue
}
vs := item.versionStatus()
switch vs {
case 0: // latest
if err := CollecDepsCheck(&item); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
hubIdx[COLLECTIONS][name] = item
}
case 1: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
default: // version is higher than the highest available from hub?
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
}
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
}
return warnings, nil
}
// Updates the info from HubInit() with the local state
func LocalSync(hub *csconfig.Hub) ([]string, error) {
skippedLocal = 0
skippedTainted = 0
warnings, err := SyncDir(hub, hub.InstallDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err)
}
_, err = SyncDir(hub, hub.HubDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err)
}
return warnings, nil
}
func GetHubIdx(hub *csconfig.Hub) error {
if hub == nil {
return fmt.Errorf("no configuration found for hub")
}
log.Debugf("loading hub idx %s", hub.HubIndexFile)
bidx, err := os.ReadFile(hub.HubIndexFile)
if err != nil {
return fmt.Errorf("unable to read index file: %w", err)
}
ret, err := LoadPkgIndex(bidx)
if err != nil {
if !errors.Is(err, ErrMissingReference) {
return fmt.Errorf("unable to load existing index: %w", err)
}
// XXX: why the error check if we bail out anyway?
return err
}
hubIdx = ret
_, err = LocalSync(hub)
if err != nil {
return fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
}
return nil
}
// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections
func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
var (
RawIndex map[string]map[string]Item
missingItems []string
)
if err := json.Unmarshal(buff, &RawIndex); err != nil {
return nil, fmt.Errorf("failed to unmarshal index: %w", err)
}
log.Debugf("%d item types in hub index", len(ItemTypes))
// Iterate over the different types to complete the struct
for _, itemType := range ItemTypes {
log.Tracef("%d item", len(RawIndex[itemType]))
for name, item := range RawIndex[itemType] {
item.Name = name
item.Type = itemType
x := strings.Split(item.RemotePath, "/")
item.FileName = x[len(x)-1]
RawIndex[itemType][name] = item
if itemType != COLLECTIONS {
continue
}
// if it's a collection, check its sub-items are present
// XXX should be done later
for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if _, ok := RawIndex[ptrtype][p]; !ok {
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
missingItems = append(missingItems, p)
}
}
}
}
}
if len(missingItems) > 0 {
return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
}
return RawIndex, nil
}

61
pkg/cwhub/remote.go Normal file
View file

@ -0,0 +1,61 @@
package cwhub
import (
"fmt"
"io"
"net/http"
)
// RemoteHubCfg is used to retrieve index and items from the remote hub.
type RemoteHubCfg struct {
Branch string
URLTemplate string
IndexPath string
}
// urlTo builds the URL to download a file from the remote hub.
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
if r == nil {
return "", ErrNilRemoteHub
}
// the template must contain two string placeholders
if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
}
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
}
// fetchIndex downloads the index from the hub and returns the content.
func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
if r == nil {
return nil, ErrNilRemoteHub
}
url, err := r.urlTo(r.IndexPath)
if err != nil {
return nil, fmt.Errorf("failed to build hub index request: %w", err)
}
resp, err := hubClient.Get(url)
if err != nil {
return nil, fmt.Errorf("failed http request for hub index: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
return nil, IndexNotFoundError{url, r.Branch}
}
return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
}
return body, nil
}

498
pkg/cwhub/sync.go Normal file
View file

@ -0,0 +1,498 @@
package cwhub
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"github.com/Masterminds/semver/v3"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)
func isYAMLFileName(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
// linkTarget returns the target of a symlink, or empty string if it's dangling.
func linkTarget(path string) (string, error) {
hubpath, err := os.Readlink(path)
if err != nil {
return "", fmt.Errorf("unable to read symlink: %s", path)
}
log.Tracef("symlink %s -> %s", path, hubpath)
_, err = os.Lstat(hubpath)
if os.IsNotExist(err) {
log.Infof("link target does not exist: %s -> %s", path, hubpath)
return "", nil
}
return hubpath, nil
}
func getSHA256(filepath string) (string, error) {
f, err := os.Open(filepath)
if err != nil {
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// information used to create a new Item, from a file path.
type itemFileInfo struct {
inhub bool
fname string
stage string
ftype string
fauthor string
}
func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
var ret *itemFileInfo
hubDir := h.local.HubDir
installDir := h.local.InstallDir
subs := strings.Split(path, string(os.PathSeparator))
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
log.Tracef("subs:%v", subs)
// we're in hub (~/.hub/hub/)
if strings.HasPrefix(path, hubDir) {
log.Tracef("in hub dir")
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml
//.../hub/profiles/crowdsec/linux.yaml
if len(subs) < 4 {
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
}
ret = &itemFileInfo{
inhub: true,
fname: subs[len(subs)-1],
fauthor: subs[len(subs)-2],
stage: subs[len(subs)-3],
ftype: subs[len(subs)-4],
}
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir")
if len(subs) < 3 {
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
}
///.../config/parser/stage/file.yaml
///.../config/postoverflow/stage/file.yaml
///.../config/scenarios/scenar.yaml
///.../config/collections/linux.yaml //file is empty
ret = &itemFileInfo{
inhub: false,
fname: subs[len(subs)-1],
stage: subs[len(subs)-2],
ftype: subs[len(subs)-3],
fauthor: "",
}
} else {
return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
}
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
if ret.stage == SCENARIOS {
ret.ftype = SCENARIOS
ret.stage = ""
} else if ret.stage == COLLECTIONS {
ret.ftype = COLLECTIONS
ret.stage = ""
} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
// it's a PARSER / POSTOVERFLOW with a stage
return nil, fmt.Errorf("unknown configuration type for file '%s'", path)
}
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
return ret, nil
}
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
func sortedVersions(raw []string) ([]string, error) {
vs := make([]*semver.Version, len(raw))
for idx, r := range raw {
v, err := semver.NewVersion(r)
if err != nil {
return nil, fmt.Errorf("%s: %w", r, err)
}
vs[idx] = v
}
sort.Sort(sort.Reverse(semver.Collection(vs)))
ret := make([]string, len(vs))
for idx, v := range vs {
ret[idx] = v.Original()
}
return ret, nil
}
func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
type localItemName struct {
Name string `yaml:"name"`
}
_, fileName := filepath.Split(path)
item := &Item{
hub: h,
Name: info.fname,
Stage: info.stage,
Type: info.ftype,
FileName: fileName,
State: ItemState{
LocalPath: path,
Installed: true,
UpToDate: true,
},
}
// try to read the name from the file
itemName := localItemName{}
itemContent, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %w", path, err)
}
err = yaml.Unmarshal(itemContent, &itemName)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
}
if itemName.Name != "" {
item.Name = itemName.Name
}
return item, nil
}
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
hubpath := ""
if err != nil {
log.Debugf("while syncing hub dir: %s", err)
// there is a path error, we ignore the file
return nil
}
// only happens if the current working directory was removed (!)
path, err = filepath.Abs(path)
if err != nil {
return err
}
// we only care about YAML files
if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) {
return nil
}
info, err := h.getItemFileInfo(path)
if err != nil {
return err
}
// non symlinks are local user files or hub files
if f.Type()&os.ModeSymlink == 0 {
log.Tracef("%s is not a symlink", path)
if !info.inhub {
log.Tracef("%s is a local file, skip", path)
item, err := newLocalItem(h, path, info)
if err != nil {
return err
}
h.Items[info.ftype][item.Name] = item
return nil
}
} else {
hubpath, err = linkTarget(path)
if err != nil {
return err
}
if hubpath == "" {
// target does not exist, the user might have removed the file
// or switched to a hub branch without it
return nil
}
}
// try to find which configuration item it is
log.Tracef("check [%s] of %s", info.fname, info.ftype)
for name, item := range h.Items[info.ftype] {
if info.fname != item.FileName {
continue
}
if item.Stage != info.stage {
continue
}
// if we are walking hub dir, just mark present files as downloaded
if info.inhub {
// wrong author
if info.fauthor != item.Author {
continue
}
// not the item we're looking for
if !item.validPath(info.fauthor, info.fname) {
continue
}
src, err := item.downloadPath()
if err != nil {
return err
}
if path == src {
log.Tracef("marking %s as downloaded", item.Name)
item.State.Downloaded = true
}
} else if !hasPathSuffix(hubpath, item.RemotePath) {
// wrong file
// <type>/<stage>/<author>/<name>.yaml
continue
}
err := item.setVersionState(path, info.inhub)
if err != nil {
return err
}
h.Items[info.ftype][name] = item
return nil
}
log.Infof("Ignoring file %s of type %s", path, info.ftype)
return nil
}
// checkSubItemVersions checks for the presence, taint and version state of sub-items.
func (i *Item) checkSubItemVersions() error {
if !i.HasSubItems() {
return nil
}
if i.versionStatus() != versionUpToDate {
log.Debugf("%s dependencies not checked: not up-to-date", i.Name)
return nil
}
// ensure all the sub-items are installed, or tag the parent as tainted
log.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed)
for _, sub := range i.SubItems() {
log.Tracef("check %s installed:%t", sub.Name, sub.State.Installed)
if !i.State.Installed {
continue
}
if err := sub.checkSubItemVersions(); err != nil {
if sub.State.Tainted {
i.State.Tainted = true
}
return fmt.Errorf("dependency of %s: sub collection %s is broken: %w", i.Name, sub.Name, err)
}
if sub.State.Tainted {
i.State.Tainted = true
return fmt.Errorf("%s is tainted because %s:%s is tainted", i.Name, sub.Type, sub.Name)
}
if !sub.State.Installed && i.State.Installed {
i.State.Tainted = true
return fmt.Errorf("%s is tainted because %s:%s is missing", i.Name, sub.Type, sub.Name)
}
if !sub.State.UpToDate {
i.State.UpToDate = false
return fmt.Errorf("dependency of %s: outdated %s:%s", i.Name, sub.Type, sub.Name)
}
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate)
}
return nil
}
// syncDir scans a directory for items, and updates the Hub state accordingly.
func (h *Hub) syncDir(dir string) error {
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
for _, scan := range ItemTypes {
// cpath: top-level item directory, either downloaded or installed items.
// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
if err != nil {
log.Errorf("failed %s: %s", cpath, err)
continue
}
// explicit check for non existing directory, avoid spamming log.Debug
if _, err = os.Stat(cpath); os.IsNotExist(err) {
log.Tracef("directory %s doesn't exist, skipping", cpath)
continue
}
if err = filepath.WalkDir(cpath, h.itemVisit); err != nil {
return err
}
}
return nil
}
// insert a string in a sorted slice, case insensitive, and return the new slice.
func insertInOrderNoCase(sl []string, value string) []string {
i := sort.Search(len(sl), func(i int) bool {
return strings.ToLower(sl[i]) >= strings.ToLower(value)
})
return append(sl[:i], append([]string{value}, sl[i:]...)...)
}
// localSync updates the hub state with downloaded, installed and local items.
func (h *Hub) localSync() error {
err := h.syncDir(h.local.InstallDir)
if err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
}
if err = h.syncDir(h.local.HubDir); err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
}
warnings := make([]string, 0)
for _, item := range h.Items[COLLECTIONS] {
// check for cyclic dependencies
subs, err := item.descendants()
if err != nil {
return err
}
// populate the sub- and sub-sub-items with the collections they belong to
for _, sub := range subs {
sub.State.BelongsToCollections = insertInOrderNoCase(sub.State.BelongsToCollections, item.Name)
}
if !item.State.Installed {
continue
}
vs := item.versionStatus()
switch vs {
case versionUpToDate: // latest
if err := item.checkSubItemVersions(); err != nil {
warnings = append(warnings, err.Error())
}
case versionUpdateAvailable: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
case versionFuture:
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
case versionUnknown:
if !item.IsLocal() {
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
}
}
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
}
h.Warnings = warnings
return nil
}
func (i *Item) setVersionState(path string, inhub bool) error {
var err error
i.State.LocalHash, err = getSHA256(path)
if err != nil {
return fmt.Errorf("failed to get sha256 of %s: %w", path, err)
}
// let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(i.Versions))
for k := range i.Versions {
versions = append(versions, k)
}
versions, err = sortedVersions(versions)
if err != nil {
return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err)
}
i.State.LocalVersion = "?"
for _, version := range versions {
if i.Versions[version].Digest == i.State.LocalHash {
i.State.LocalVersion = version
break
}
}
if i.State.LocalVersion == "?" {
log.Tracef("got tainted match for %s: %s", i.Name, path)
if !inhub {
i.State.LocalPath = path
i.State.Installed = true
}
i.State.UpToDate = false
i.State.Tainted = true
return nil
}
// we got an exact match, update struct
i.State.Downloaded = true
if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version)
i.State.LocalPath = path
i.State.Tainted = false
// if we're walking the hub, present file doesn't means installed file
i.State.Installed = true
}
if i.State.LocalVersion == i.Version {
log.Tracef("%s is up-to-date", i.Name)
i.State.UpToDate = true
}
return nil
}

147
pkg/hubtest/coverage.go
View file

@ -5,173 +5,194 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"sort"
"strings" "strings"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
) )
type ParserCoverage struct { type Coverage struct {
Parser string Name string
TestsCount int TestsCount int
PresentIn map[string]bool //poorman's set PresentIn map[string]bool //poorman's set
} }
type ScenarioCoverage struct { func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
Scenario string if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
TestsCount int return nil, fmt.Errorf("no parsers in hub index")
PresentIn map[string]bool }
}
func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) { // populate from hub, iterate in alphabetical order
var coverage []ParserCoverage pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS])
if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok { coverage := make([]Coverage, len(pkeys))
return coverage, fmt.Errorf("no parsers in hub index")
} for i, name := range pkeys {
//populate from hub, iterate in alphabetical order coverage[i] = Coverage{
var pkeys []string Name: name,
for pname := range h.HubIndex.Data[cwhub.PARSERS] {
pkeys = append(pkeys, pname)
}
sort.Strings(pkeys)
for _, pname := range pkeys {
coverage = append(coverage, ParserCoverage{
Parser: pname,
TestsCount: 0, TestsCount: 0,
PresentIn: make(map[string]bool), PresentIn: make(map[string]bool),
}) }
} }
//parser the expressions a-la-oneagain // parser the expressions a-la-oneagain
passerts, err := filepath.Glob(".tests/*/parser.assert") passerts, err := filepath.Glob(".tests/*/parser.assert")
if err != nil { if err != nil {
return coverage, fmt.Errorf("while find parser asserts : %s", err) return nil, fmt.Errorf("while find parser asserts : %s", err)
} }
for _, assert := range passerts { for _, assert := range passerts {
file, err := os.Open(assert) file, err := os.Open(assert)
if err != nil { if err != nil {
return coverage, fmt.Errorf("while reading %s : %s", assert, err) return nil, fmt.Errorf("while reading %s : %s", assert, err)
} }
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
for scanner.Scan() { for scanner.Scan() {
assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
line := scanner.Text() line := scanner.Text()
log.Debugf("assert line : %s", line) log.Debugf("assert line : %s", line)
match := assertLine.FindStringSubmatch(line)
match := parserResultRE.FindStringSubmatch(line)
if len(match) == 0 { if len(match) == 0 {
log.Debugf("%s doesn't match", line) log.Debugf("%s doesn't match", line)
continue continue
} }
sidx := assertLine.SubexpIndex("parser")
sidx := parserResultRE.SubexpIndex("parser")
capturedParser := match[sidx] capturedParser := match[sidx]
for idx, pcover := range coverage { for idx, pcover := range coverage {
if pcover.Parser == capturedParser { if pcover.Name == capturedParser {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
parserNameSplit := strings.Split(pcover.Parser, "/")
parserNameSplit := strings.Split(pcover.Name, "/")
parserNameOnly := parserNameSplit[len(parserNameSplit)-1] parserNameOnly := parserNameSplit[len(parserNameSplit)-1]
if parserNameOnly == capturedParser { if parserNameOnly == capturedParser {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
capturedParserSplit := strings.Split(capturedParser, "/") capturedParserSplit := strings.Split(capturedParser, "/")
capturedParserName := capturedParserSplit[len(capturedParserSplit)-1] capturedParserName := capturedParserSplit[len(capturedParserSplit)-1]
if capturedParserName == parserNameOnly { if capturedParserName == parserNameOnly {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
if capturedParserName == parserNameOnly+"-logs" { if capturedParserName == parserNameOnly+"-logs" {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
} }
} }
file.Close() file.Close()
} }
return coverage, nil return coverage, nil
} }
func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) { func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
var coverage []ScenarioCoverage if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok { return nil, fmt.Errorf("no scenarios in hub index")
return coverage, fmt.Errorf("no scenarios in hub index")
}
//populate from hub, iterate in alphabetical order
var pkeys []string
for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] {
pkeys = append(pkeys, scenarioName)
}
sort.Strings(pkeys)
for _, scenarioName := range pkeys {
coverage = append(coverage, ScenarioCoverage{
Scenario: scenarioName,
TestsCount: 0,
PresentIn: make(map[string]bool),
})
} }
//parser the expressions a-la-oneagain // populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS])
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {
coverage[i] = Coverage{
Name: name,
TestsCount: 0,
PresentIn: make(map[string]bool),
}
}
// parser the expressions a-la-oneagain
passerts, err := filepath.Glob(".tests/*/scenario.assert") passerts, err := filepath.Glob(".tests/*/scenario.assert")
if err != nil { if err != nil {
return coverage, fmt.Errorf("while find scenario asserts : %s", err) return nil, fmt.Errorf("while find scenario asserts : %s", err)
} }
for _, assert := range passerts { for _, assert := range passerts {
file, err := os.Open(assert) file, err := os.Open(assert)
if err != nil { if err != nil {
return coverage, fmt.Errorf("while reading %s : %s", assert, err) return nil, fmt.Errorf("while reading %s : %s", assert, err)
} }
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
for scanner.Scan() { for scanner.Scan() {
assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
line := scanner.Text() line := scanner.Text()
log.Debugf("assert line : %s", line) log.Debugf("assert line : %s", line)
match := assertLine.FindStringSubmatch(line) match := scenarioResultRE.FindStringSubmatch(line)
if len(match) == 0 { if len(match) == 0 {
log.Debugf("%s doesn't match", line) log.Debugf("%s doesn't match", line)
continue continue
} }
sidx := assertLine.SubexpIndex("scenario")
scanner_name := match[sidx] sidx := scenarioResultRE.SubexpIndex("scenario")
scannerName := match[sidx]
for idx, pcover := range coverage { for idx, pcover := range coverage {
if pcover.Scenario == scanner_name { if pcover.Name == scannerName {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
scenarioNameSplit := strings.Split(pcover.Scenario, "/")
scenarioNameSplit := strings.Split(pcover.Name, "/")
scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1] scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1]
if scenarioNameOnly == scanner_name {
if scenarioNameOnly == scannerName {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
fixedProbingWord := strings.ReplaceAll(pcover.Scenario, "probbing", "probing")
fixedProbingAssert := strings.ReplaceAll(scanner_name, "probbing", "probing") fixedProbingWord := strings.ReplaceAll(pcover.Name, "probbing", "probing")
fixedProbingAssert := strings.ReplaceAll(scannerName, "probbing", "probing")
if fixedProbingWord == fixedProbingAssert { if fixedProbingWord == fixedProbingAssert {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name {
if fmt.Sprintf("%s-detection", pcover.Name) == scannerName {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert { if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert {
coverage[idx].TestsCount++ coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true coverage[idx].PresentIn[assert] = true
continue continue
} }
} }
} }
file.Close() file.Close()
} }
return coverage, nil return coverage, nil
} }

38
pkg/hubtest/hubtest.go
View file

@ -6,6 +6,7 @@ import (
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
) )
@ -18,7 +19,7 @@ type HubTest struct {
TemplateConfigPath string TemplateConfigPath string
TemplateProfilePath string TemplateProfilePath string
TemplateSimulationPath string TemplateSimulationPath string
HubIndex *HubIndex HubIndex *cwhub.Hub
Tests []*HubTestItem Tests []*HubTestItem
} }
@ -29,42 +30,44 @@ const (
) )
func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) { func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) {
var err error hubPath, err := filepath.Abs(hubPath)
hubPath, err = filepath.Abs(hubPath)
if err != nil { if err != nil {
return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err) return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err)
} }
// we can't use hubtest without the hub // we can't use hubtest without the hub
if _, err := os.Stat(hubPath); os.IsNotExist(err) { if _, err = os.Stat(hubPath); os.IsNotExist(err) {
return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath) return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath)
} }
HubTestPath := filepath.Join(hubPath, "./.tests/") HubTestPath := filepath.Join(hubPath, "./.tests/")
// we can't use hubtest without crowdsec binary // we can't use hubtest without crowdsec binary
if _, err := exec.LookPath(crowdsecPath); err != nil { if _, err = exec.LookPath(crowdsecPath); err != nil {
if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) { if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) {
return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath) return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath)
} }
} }
// we can't use hubtest without cscli binary // we can't use hubtest without cscli binary
if _, err := exec.LookPath(cscliPath); err != nil { if _, err = exec.LookPath(cscliPath); err != nil {
if _, err := os.Stat(cscliPath); os.IsNotExist(err) { if _, err = os.Stat(cscliPath); os.IsNotExist(err) {
return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath) return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath)
} }
} }
hubIndexFile := filepath.Join(hubPath, ".index.json") hubIndexFile := filepath.Join(hubPath, ".index.json")
bidx, err := os.ReadFile(hubIndexFile)
if err != nil { local := &csconfig.LocalHubCfg{
return HubTest{}, fmt.Errorf("unable to read index file: %s", err) HubDir: hubPath,
HubIndexFile: hubIndexFile,
InstallDir: HubTestPath,
InstallDataDir: HubTestPath,
} }
// load hub index hub, err := cwhub.NewHub(local, nil, false)
hubIndex, err := cwhub.LoadPkgIndex(bidx)
if err != nil { if err != nil {
return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err) return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
} }
templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile) templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile)
@ -80,16 +83,18 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
TemplateConfigPath: templateConfigFilePath, TemplateConfigPath: templateConfigFilePath,
TemplateProfilePath: templateProfilePath, TemplateProfilePath: templateProfilePath,
TemplateSimulationPath: templateSimulationPath, TemplateSimulationPath: templateSimulationPath,
HubIndex: &HubIndex{Data: hubIndex}, HubIndex: hub,
}, nil }, nil
} }
func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) { func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) {
HubTestItem := &HubTestItem{} HubTestItem := &HubTestItem{}
testItem, err := NewTest(name, h) testItem, err := NewTest(name, h)
if err != nil { if err != nil {
return HubTestItem, err return HubTestItem, err
} }
h.Tests = append(h.Tests, testItem) h.Tests = append(h.Tests, testItem)
return testItem, nil return testItem, nil
@ -108,5 +113,6 @@ func (h *HubTest) LoadAllTests() error {
} }
} }
} }
return nil return nil
} }

118
pkg/hubtest/hubtest_item.go
View file

@ -7,11 +7,12 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/parser"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
) )
type HubTestItemConfig struct { type HubTestItemConfig struct {
@ -25,10 +26,6 @@ type HubTestItemConfig struct {
OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
} }
type HubIndex struct {
Data map[string]map[string]cwhub.Item
}
type HubTestItem struct { type HubTestItem struct {
Name string Name string
Path string Path string
@ -43,7 +40,7 @@ type HubTestItem struct {
RuntimeConfigFilePath string RuntimeConfigFilePath string
RuntimeProfileFilePath string RuntimeProfileFilePath string
RuntimeSimulationFilePath string RuntimeSimulationFilePath string
RuntimeHubConfig *csconfig.Hub RuntimeHubConfig *csconfig.LocalHubCfg
ResultsPath string ResultsPath string
ParserResultFile string ParserResultFile string
@ -56,7 +53,7 @@ type HubTestItem struct {
TemplateConfigPath string TemplateConfigPath string
TemplateProfilePath string TemplateProfilePath string
TemplateSimulationPath string TemplateSimulationPath string
HubIndex *HubIndex HubIndex *cwhub.Hub
Config *HubTestItemConfig Config *HubTestItemConfig
@ -80,8 +77,6 @@ const (
BucketPourResultFileName = "bucketpour-dump.yaml" BucketPourResultFileName = "bucketpour-dump.yaml"
) )
var crowdsecPatternsFolder = csconfig.DefaultConfigPath("patterns")
func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) { func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
testPath := filepath.Join(hubTest.HubTestPath, name) testPath := filepath.Join(hubTest.HubTestPath, name)
runtimeFolder := filepath.Join(testPath, "runtime") runtimeFolder := filepath.Join(testPath, "runtime")
@ -91,10 +86,12 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
// read test configuration file // read test configuration file
configFileData := &HubTestItemConfig{} configFileData := &HubTestItemConfig{}
yamlFile, err := os.ReadFile(configFilePath) yamlFile, err := os.ReadFile(configFilePath)
if err != nil { if err != nil {
log.Printf("no config file found in '%s': %v", testPath, err) log.Printf("no config file found in '%s': %v", testPath, err)
} }
err = yaml.Unmarshal(yamlFile, configFileData) err = yaml.Unmarshal(yamlFile, configFileData)
if err != nil { if err != nil {
return nil, fmt.Errorf("unmarshal: %v", err) return nil, fmt.Errorf("unmarshal: %v", err)
@ -105,6 +102,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName) scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName)
ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath) ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath)
return &HubTestItem{ return &HubTestItem{
Name: name, Name: name,
Path: testPath, Path: testPath,
@ -121,7 +119,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
ParserResultFile: filepath.Join(resultPath, ParserResultFileName), ParserResultFile: filepath.Join(resultPath, ParserResultFileName),
ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName), ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName),
BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName), BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName),
RuntimeHubConfig: &csconfig.Hub{ RuntimeHubConfig: &csconfig.LocalHubCfg{
HubDir: runtimeHubFolder, HubDir: runtimeHubFolder,
HubIndexFile: hubTest.HubIndexFile, HubIndexFile: hubTest.HubIndexFile,
InstallDir: runtimeFolder, InstallDir: runtimeFolder,
@ -147,23 +145,25 @@ func (t *HubTestItem) InstallHub() error {
if parser == "" { if parser == "" {
continue continue
} }
var parserDirDest string
if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok { if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath)) parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
if err != nil { if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err) return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
} }
parserFileName := filepath.Base(parserSource) parserFileName := filepath.Base(parserSource)
// runtime/hub/parsers/s00-raw/crowdsecurity/ // runtime/hub/parsers/s00-raw/crowdsecurity/
hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath)) hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath))
// runtime/parsers/s00-raw/ // runtime/parsers/s00-raw/
parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage) parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil { if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err) return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err)
} }
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
} }
@ -204,7 +204,7 @@ func (t *HubTestItem) InstallHub() error {
//return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath) //return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
} }
parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage) parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil { if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
continue continue
//return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err) //return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
@ -231,23 +231,25 @@ func (t *HubTestItem) InstallHub() error {
if scenario == "" { if scenario == "" {
continue continue
} }
var scenarioDirDest string
if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok { if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath)) scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
if err != nil { if err != nil {
return fmt.Errorf("can't get absolute path to: %s", scenarioSource) return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
} }
scenarioFileName := filepath.Base(scenarioSource) scenarioFileName := filepath.Base(scenarioSource)
// runtime/hub/scenarios/crowdsecurity/ // runtime/hub/scenarios/crowdsecurity/
hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath)) hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath))
// runtime/parsers/scenarios/ // runtime/parsers/scenarios/
scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil { if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err) return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err)
} }
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
} }
@ -275,7 +277,7 @@ func (t *HubTestItem) InstallHub() error {
//return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario) //return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario)
} }
scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath) scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil { if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err) return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
} }
@ -300,23 +302,25 @@ func (t *HubTestItem) InstallHub() error {
if postoverflow == "" { if postoverflow == "" {
continue continue
} }
var postoverflowDirDest string
if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok { if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath)) postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
if err != nil { if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err) return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
} }
postoverflowFileName := filepath.Base(postoverflowSource) postoverflowFileName := filepath.Base(postoverflowSource)
// runtime/hub/postoverflows/s00-enrich/crowdsecurity/ // runtime/hub/postoverflows/s00-enrich/crowdsecurity/
hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath)) hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath))
// runtime/postoverflows/s00-enrich // runtime/postoverflows/s00-enrich
postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage) postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil { if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err) return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err)
} }
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
} }
@ -357,7 +361,7 @@ func (t *HubTestItem) InstallHub() error {
//return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath) //return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
} }
postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage) postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil { if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
continue continue
//return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err) //return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
@ -384,10 +388,12 @@ func (t *HubTestItem) InstallHub() error {
Filter: "1==1", Filter: "1==1",
Statics: t.Config.OverrideStatics, Statics: t.Config.OverrideStatics,
} }
b, err := yaml.Marshal(n) b, err := yaml.Marshal(n)
if err != nil { if err != nil {
return fmt.Errorf("unable to marshal overrides: %s", err) return fmt.Errorf("unable to marshal overrides: %s", err)
} }
tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath) tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath)
if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil { if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil {
return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err) return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err)
@ -395,40 +401,43 @@ func (t *HubTestItem) InstallHub() error {
} }
// load installed hub // load installed hub
err := cwhub.GetHubIdx(t.RuntimeHubConfig) hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false)
if err != nil { if err != nil {
log.Fatalf("can't local sync the hub: %+v", err) log.Fatal(err)
} }
// install data for parsers if needed // install data for parsers if needed
ret := cwhub.GetItemMap(cwhub.PARSERS) ret := hub.GetItemMap(cwhub.PARSERS)
for parserName, item := range ret { for parserName, item := range ret {
if item.Installed { if item.State.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
} }
log.Debugf("parser '%s' installed successfully in runtime environment", parserName) log.Debugf("parser '%s' installed successfully in runtime environment", parserName)
} }
} }
// install data for scenarios if needed // install data for scenarios if needed
ret = cwhub.GetItemMap(cwhub.SCENARIOS) ret = hub.GetItemMap(cwhub.SCENARIOS)
for scenarioName, item := range ret { for scenarioName, item := range ret {
if item.Installed { if item.State.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
} }
log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName) log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName)
} }
} }
// install data for postoverflows if needed // install data for postoverflows if needed
ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW) ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
for postoverflowName, item := range ret { for postoverflowName, item := range ret {
if item.Installed { if item.State.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
} }
log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName) log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName)
} }
} }
@ -455,51 +464,53 @@ func (t *HubTestItem) Run() error {
} }
// create runtime folder // create runtime folder
if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil { if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err) return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
} }
// create runtime data folder // create runtime data folder
if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil { if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err) return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
} }
// create runtime hub folder // create runtime hub folder
if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil { if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err) return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
} }
if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil { if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err) return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
} }
// create results folder // create results folder
if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil { if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err) return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
} }
// copy template config file to runtime folder // copy template config file to runtime folder
if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil { if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err) return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
} }
// copy template profile file to runtime folder // copy template profile file to runtime folder
if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil { if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err) return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
} }
// copy template simulation file to runtime folder // copy template simulation file to runtime folder
if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil { if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err) return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
} }
crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns")
// copy template patterns folder to runtime folder // copy template patterns folder to runtime folder
if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil { if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err) return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
} }
// install the hub in the runtime folder // install the hub in the runtime folder
if err := t.InstallHub(); err != nil { if err = t.InstallHub(); err != nil {
return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err) return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
} }
@ -507,7 +518,7 @@ func (t *HubTestItem) Run() error {
logType := t.Config.LogType logType := t.Config.LogType
dsn := fmt.Sprintf("file://%s", logFile) dsn := fmt.Sprintf("file://%s", logFile)
if err := os.Chdir(testPath); err != nil { if err = os.Chdir(testPath); err != nil {
return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err) return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
} }
@ -515,6 +526,7 @@ func (t *HubTestItem) Run() error {
if err != nil { if err != nil {
return fmt.Errorf("unable to stat log file '%s': %s", logFile, err) return fmt.Errorf("unable to stat log file '%s': %s", logFile, err)
} }
if logFileStat.Size() == 0 { if logFileStat.Size() == 0 {
return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile) return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile)
} }
@ -522,6 +534,7 @@ func (t *HubTestItem) Run() error {
cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"} cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...) cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
log.Debugf("%s", cscliRegisterCmd.String()) log.Debugf("%s", cscliRegisterCmd.String())
output, err := cscliRegisterCmd.CombinedOutput() output, err := cscliRegisterCmd.CombinedOutput()
if err != nil { if err != nil {
if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") { if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
@ -531,16 +544,20 @@ func (t *HubTestItem) Run() error {
} }
cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"} cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"}
for labelKey, labelValue := range t.Config.Labels { for labelKey, labelValue := range t.Config.Labels {
arg := fmt.Sprintf("%s:%s", labelKey, labelValue) arg := fmt.Sprintf("%s:%s", labelKey, labelValue)
cmdArgs = append(cmdArgs, "-label", arg) cmdArgs = append(cmdArgs, "-label", arg)
} }
crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...) crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...)
log.Debugf("%s", crowdsecCmd.String()) log.Debugf("%s", crowdsecCmd.String())
output, err = crowdsecCmd.CombinedOutput() output, err = crowdsecCmd.CombinedOutput()
if log.GetLevel() >= log.DebugLevel || err != nil { if log.GetLevel() >= log.DebugLevel || err != nil {
fmt.Println(string(output)) fmt.Println(string(output))
} }
if err != nil { if err != nil {
return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err) return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err)
} }
@ -557,8 +574,10 @@ func (t *HubTestItem) Run() error {
if err != nil { if err != nil {
return err return err
} }
parserAssertFile.Close() parserAssertFile.Close()
} }
assertFileStat, err := os.Stat(t.ParserAssert.File) assertFileStat, err := os.Stat(t.ParserAssert.File)
if err != nil { if err != nil {
return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err) return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err)
@ -569,6 +588,7 @@ func (t *HubTestItem) Run() error {
if err != nil { if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err) return fmt.Errorf("couldn't generate assertion: %s", err)
} }
t.ParserAssert.AutoGenAssertData = assertData t.ParserAssert.AutoGenAssertData = assertData
t.ParserAssert.AutoGenAssert = true t.ParserAssert.AutoGenAssert = true
} else { } else {
@ -580,12 +600,15 @@ func (t *HubTestItem) Run() error {
// assert scenarios // assert scenarios
nbScenario := 0 nbScenario := 0
for _, scenario := range t.Config.Scenarios { for _, scenario := range t.Config.Scenarios {
if scenario == "" { if scenario == "" {
continue continue
} }
nbScenario += 1
nbScenario++
} }
if nbScenario > 0 { if nbScenario > 0 {
_, err := os.Stat(t.ScenarioAssert.File) _, err := os.Stat(t.ScenarioAssert.File)
if os.IsNotExist(err) { if os.IsNotExist(err) {
@ -593,8 +616,10 @@ func (t *HubTestItem) Run() error {
if err != nil { if err != nil {
return err return err
} }
scenarioAssertFile.Close() scenarioAssertFile.Close()
} }
assertFileStat, err := os.Stat(t.ScenarioAssert.File) assertFileStat, err := os.Stat(t.ScenarioAssert.File)
if err != nil { if err != nil {
return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err) return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err)
@ -605,6 +630,7 @@ func (t *HubTestItem) Run() error {
if err != nil { if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err) return fmt.Errorf("couldn't generate assertion: %s", err)
} }
t.ScenarioAssert.AutoGenAssertData = assertData t.ScenarioAssert.AutoGenAssertData = assertData
t.ScenarioAssert.AutoGenAssert = true t.ScenarioAssert.AutoGenAssert = true
} else { } else {

174
pkg/hubtest/parser_assert.go
View file

@ -5,13 +5,11 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"regexp"
"sort" "sort"
"strings" "strings"
"time" "time"
"github.com/antonmedv/expr" "github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/enescakir/emoji" "github.com/enescakir/emoji"
"github.com/fatih/color" "github.com/fatih/color"
diff "github.com/r3labs/diff/v2" diff "github.com/r3labs/diff/v2"
@ -43,10 +41,10 @@ type ParserResult struct {
Evt types.Event Evt types.Event
Success bool Success bool
} }
type ParserResults map[string]map[string][]ParserResult type ParserResults map[string]map[string][]ParserResult
func NewParserAssert(file string) *ParserAssert { func NewParserAssert(file string) *ParserAssert {
ParserAssert := &ParserAssert{ ParserAssert := &ParserAssert{
File: file, File: file,
NbAssert: 0, NbAssert: 0,
@ -55,6 +53,7 @@ func NewParserAssert(file string) *ParserAssert {
AutoGenAssert: false, AutoGenAssert: false,
TestData: &ParserResults{}, TestData: &ParserResults{},
} }
return ParserAssert return ParserAssert
} }
@ -63,22 +62,24 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
ret := p.AutoGenParserAssert() ret := p.AutoGenParserAssert()
return ret, nil return ret, nil
} }
func (p *ParserAssert) LoadTest(filename string) error { func (p *ParserAssert) LoadTest(filename string) error {
var err error
parserDump, err := LoadParserDump(filename) parserDump, err := LoadParserDump(filename)
if err != nil { if err != nil {
return fmt.Errorf("loading parser dump file: %+v", err) return fmt.Errorf("loading parser dump file: %+v", err)
} }
p.TestData = parserDump p.TestData = parserDump
return nil return nil
} }
func (p *ParserAssert) AssertFile(testFile string) error { func (p *ParserAssert) AssertFile(testFile string) error {
file, err := os.Open(p.File) file, err := os.Open(p.File)
if err != nil { if err != nil {
@ -88,19 +89,26 @@ func (p *ParserAssert) AssertFile(testFile string) error {
if err := p.LoadTest(testFile); err != nil { if err := p.LoadTest(testFile); err != nil {
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
} }
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines) scanner.Split(bufio.ScanLines)
nbLine := 0 nbLine := 0
for scanner.Scan() { for scanner.Scan() {
nbLine += 1 nbLine++
if scanner.Text() == "" { if scanner.Text() == "" {
continue continue
} }
ok, err := p.Run(scanner.Text()) ok, err := p.Run(scanner.Text())
if err != nil { if err != nil {
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
} }
p.NbAssert += 1
p.NbAssert++
if !ok { if !ok {
log.Debugf("%s is FALSE", scanner.Text()) log.Debugf("%s is FALSE", scanner.Text())
failedAssert := &AssertFail{ failedAssert := &AssertFail{
@ -109,37 +117,43 @@ func (p *ParserAssert) AssertFile(testFile string) error {
Expression: scanner.Text(), Expression: scanner.Text(),
Debug: make(map[string]string), Debug: make(map[string]string),
} }
variableRE := regexp.MustCompile(`(?P<variable>[^ =]+) == .*`)
match := variableRE.FindStringSubmatch(scanner.Text()) match := variableRE.FindStringSubmatch(scanner.Text())
variable := "" variable := ""
if len(match) == 0 { if len(match) == 0 {
log.Infof("Couldn't get variable of line '%s'", scanner.Text()) log.Infof("Couldn't get variable of line '%s'", scanner.Text())
variable = scanner.Text() variable = scanner.Text()
} else { } else {
variable = match[1] variable = match[1]
} }
result, err := p.EvalExpression(variable) result, err := p.EvalExpression(variable)
if err != nil { if err != nil {
log.Errorf("unable to evaluate variable '%s': %s", variable, err) log.Errorf("unable to evaluate variable '%s': %s", variable, err)
continue continue
} }
failedAssert.Debug[variable] = result failedAssert.Debug[variable] = result
p.Fails = append(p.Fails, *failedAssert) p.Fails = append(p.Fails, *failedAssert)
continue continue
} }
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
} }
file.Close() file.Close()
if p.NbAssert == 0 { if p.NbAssert == 0 {
assertData, err := p.AutoGenFromFile(testFile) assertData, err := p.AutoGenFromFile(testFile)
if err != nil { if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err) return fmt.Errorf("couldn't generate assertion: %s", err)
} }
p.AutoGenAssertData = assertData p.AutoGenAssertData = assertData
p.AutoGenAssert = true p.AutoGenAssert = true
} }
if len(p.Fails) == 0 { if len(p.Fails) == 0 {
p.Success = true p.Success = true
} }
@ -148,15 +162,14 @@ func (p *ParserAssert) AssertFile(testFile string) error {
} }
func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
var err error
//debug doesn't make much sense with the ability to evaluate "on the fly" //debug doesn't make much sense with the ability to evaluate "on the fly"
//var debugFilter *exprhelpers.ExprDebugger //var debugFilter *exprhelpers.ExprDebugger
var runtimeFilter *vm.Program
var output interface{} var output interface{}
env := map[string]interface{}{"results": *p.TestData} env := map[string]interface{}{"results": *p.TestData}
if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
if err != nil {
log.Errorf("failed to compile '%s' : %s", expression, err) log.Errorf("failed to compile '%s' : %s", expression, err)
return output, err return output, err
} }
@ -168,8 +181,10 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
if err != nil { if err != nil {
log.Warningf("running : %s", expression) log.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err) log.Warningf("runtime error : %s", err)
return output, fmt.Errorf("while running expression %s: %w", expression, err) return output, fmt.Errorf("while running expression %s: %w", expression, err)
} }
return output, nil return output, nil
} }
@ -178,10 +193,13 @@ func (p *ParserAssert) EvalExpression(expression string) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
ret, err := yaml.Marshal(output) ret, err := yaml.Marshal(output)
if err != nil { if err != nil {
return "", err return "", err
} }
return string(ret), nil return string(ret), nil
} }
@ -190,6 +208,7 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
if err != nil { if err != nil {
return false, err return false, err
} }
switch out := output.(type) { switch out := output.(type) {
case bool: case bool:
return out, nil return out, nil
@ -201,80 +220,89 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
func Escape(val string) string { func Escape(val string) string {
val = strings.ReplaceAll(val, `\`, `\\`) val = strings.ReplaceAll(val, `\`, `\\`)
val = strings.ReplaceAll(val, `"`, `\"`) val = strings.ReplaceAll(val, `"`, `\"`)
return val return val
} }
func (p *ParserAssert) AutoGenParserAssert() string { func (p *ParserAssert) AutoGenParserAssert() string {
//attempt to autogen parser asserts //attempt to autogen parser asserts
var ret string ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
//sort map keys for consistent order
stages := sortedMapKeys(*p.TestData)
//sort map keys for consistent ordre
var stages []string
for stage := range *p.TestData {
stages = append(stages, stage)
}
sort.Strings(stages)
ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
for _, stage := range stages { for _, stage := range stages {
parsers := (*p.TestData)[stage] parsers := (*p.TestData)[stage]
//sort map keys for consistent ordre
var pnames []string //sort map keys for consistent order
for pname := range parsers { pnames := sortedMapKeys(parsers)
pnames = append(pnames, pname)
}
sort.Strings(pnames)
for _, parser := range pnames { for _, parser := range pnames {
presults := parsers[parser] presults := parsers[parser]
ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults)) ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults))
for pidx, result := range presults { for pidx, result := range presults {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success) ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success)
if !result.Success { if !result.Success {
continue continue
} }
for _, pkey := range sortedMapKeys(result.Evt.Parsed) { for _, pkey := range sortedMapKeys(result.Evt.Parsed) {
pval := result.Evt.Parsed[pkey] pval := result.Evt.Parsed[pkey]
if pval == "" { if pval == "" {
continue continue
} }
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval)) ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
} }
for _, mkey := range sortedMapKeys(result.Evt.Meta) { for _, mkey := range sortedMapKeys(result.Evt.Meta) {
mval := result.Evt.Meta[mkey] mval := result.Evt.Meta[mkey]
if mval == "" { if mval == "" {
continue continue
} }
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval)) ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
} }
for _, ekey := range sortedMapKeys(result.Evt.Enriched) { for _, ekey := range sortedMapKeys(result.Evt.Enriched) {
eval := result.Evt.Enriched[ekey] eval := result.Evt.Enriched[ekey]
if eval == "" { if eval == "" {
continue continue
} }
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval)) ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
} }
for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) { for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) {
uval := result.Evt.Unmarshaled[ukey] uval := result.Evt.Unmarshaled[ukey]
if uval == "" { if uval == "" {
continue continue
} }
base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey) base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey)
for _, line := range p.buildUnmarshaledAssert(base, uval) { for _, line := range p.buildUnmarshaledAssert(base, uval) {
ret += line ret += line
} }
} }
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted) ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted)
if result.Evt.WhitelistReason != "" { if result.Evt.WhitelistReason != "" {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason)) ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason))
} }
} }
} }
} }
return ret return ret
} }
func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string { func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string {
ret := make([]string, 0) ret := make([]string, 0)
switch val := eval.(type) { switch val := eval.(type) {
case map[string]interface{}: case map[string]interface{}:
for k, v := range val { for k, v := range val {
@ -297,12 +325,11 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s
default: default:
log.Warningf("unknown type '%T' for key '%s'", val, ekey) log.Warningf("unknown type '%T' for key '%s'", val, ekey)
} }
return ret return ret
} }
func LoadParserDump(filepath string) (*ParserResults, error) { func LoadParserDump(filepath string) (*ParserResults, error) {
var pdump ParserResults
dumpData, err := os.Open(filepath) dumpData, err := os.Open(filepath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -314,18 +341,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
return nil, err return nil, err
} }
pdump := ParserResults{}
if err := yaml.Unmarshal(results, &pdump); err != nil { if err := yaml.Unmarshal(results, &pdump); err != nil {
return nil, err return nil, err
} }
/* we know that some variables should always be set, /* we know that some variables should always be set,
let's check if they're present in last parser output of last stage */ let's check if they're present in last parser output of last stage */
stages := make([]string, 0, len(pdump))
for k := range pdump { stages := sortedMapKeys(pdump)
stages = append(stages, k)
}
sort.Strings(stages)
var lastStage string var lastStage string
//Loop over stages to find last successful one with at least one parser //Loop over stages to find last successful one with at least one parser
for i := len(stages) - 2; i >= 0; i-- { for i := len(stages) - 2; i >= 0; i-- {
if len(pdump[stages[i]]) != 0 { if len(pdump[stages[i]]) != 0 {
@ -333,11 +361,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
break break
} }
} }
parsers := make([]string, 0, len(pdump[lastStage])) parsers := make([]string, 0, len(pdump[lastStage]))
for k := range pdump[lastStage] { for k := range pdump[lastStage] {
parsers = append(parsers, k) parsers = append(parsers, k)
} }
sort.Strings(parsers) sort.Strings(parsers)
if len(parsers) == 0 {
return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
}
lastParser := parsers[len(parsers)-1] lastParser := parsers[len(parsers)-1]
for idx, result := range pdump[lastStage][lastParser] { for idx, result := range pdump[lastStage][lastParser] {
@ -357,47 +393,51 @@ type DumpOpts struct {
ShowNotOkParsers bool ShowNotOkParsers bool
} }
func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts DumpOpts) { func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
//note : we can use line -> time as the unique identifier (of acquisition) //note : we can use line -> time as the unique identifier (of acquisition)
state := make(map[time.Time]map[string]map[string]ParserResult) state := make(map[time.Time]map[string]map[string]ParserResult)
assoc := make(map[time.Time]string, 0) assoc := make(map[time.Time]string, 0)
for stage, parsers := range parser_results { for stage, parsers := range parserResults {
for parser, results := range parsers { for parser, results := range parsers {
for _, parser_res := range results { for _, parserRes := range results {
evt := parser_res.Evt evt := parserRes.Evt
if _, ok := state[evt.Line.Time]; !ok { if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult) state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw assoc[evt.Line.Time] = evt.Line.Raw
} }
if _, ok := state[evt.Line.Time][stage]; !ok { if _, ok := state[evt.Line.Time][stage]; !ok {
state[evt.Line.Time][stage] = make(map[string]ParserResult) state[evt.Line.Time][stage] = make(map[string]ParserResult)
} }
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success}
}
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
}
} }
} }
for bname, evtlist := range bucket_pour { for bname, evtlist := range bucketPour {
for _, evt := range evtlist { for _, evt := range evtlist {
if evt.Line.Raw == "" { if evt.Line.Raw == "" {
continue continue
} }
//it might be bucket overflow being reprocessed, skip this //it might be bucket overflow being reprocessed, skip this
if _, ok := state[evt.Line.Time]; !ok { if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult) state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw assoc[evt.Line.Time] = evt.Line.Raw
} }
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if _, ok := state[evt.Line.Time]["buckets"]; !ok { if _, ok := state[evt.Line.Time]["buckets"]; !ok {
state[evt.Line.Time]["buckets"] = make(map[string]ParserResult) state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
} }
state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true} state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
} }
} }
yellow := color.New(color.FgYellow).SprintFunc() yellow := color.New(color.FgYellow).SprintFunc()
red := color.New(color.FgRed).SprintFunc() red := color.New(color.FgRed).SprintFunc()
green := color.New(color.FgGreen).SprintFunc() green := color.New(color.FgGreen).SprintFunc()
@ -409,19 +449,25 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
continue continue
} }
} }
fmt.Printf("line: %s\n", rawstr) fmt.Printf("line: %s\n", rawstr)
skeys := make([]string, 0, len(state[tstamp])) skeys := make([]string, 0, len(state[tstamp]))
for k := range state[tstamp] { for k := range state[tstamp] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "buckets" { if k == "buckets" {
continue continue
} }
skeys = append(skeys, k) skeys = append(skeys, k)
} }
sort.Strings(skeys) sort.Strings(skeys)
//iterate stage
var prev_item types.Event // iterate stage
var prevItem types.Event
for _, stage := range skeys { for _, stage := range skeys {
parsers := state[tstamp][stage] parsers := state[tstamp][stage]
@ -431,18 +477,16 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
fmt.Printf("\t%s %s\n", sep, stage) fmt.Printf("\t%s %s\n", sep, stage)
pkeys := make([]string, 0, len(parsers)) pkeys := sortedMapKeys(parsers)
for k := range parsers {
pkeys = append(pkeys, k)
}
sort.Strings(pkeys)
for idx, parser := range pkeys { for idx, parser := range pkeys {
res := parsers[parser].Success res := parsers[parser].Success
sep := "├" sep := "├"
if idx == len(pkeys)-1 { if idx == len(pkeys)-1 {
sep = "└" sep = "└"
} }
created := 0 created := 0
updated := 0 updated := 0
deleted := 0 deleted := 0
@ -451,16 +495,19 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
detailsDisplay := "" detailsDisplay := ""
if res { if res {
changelog, _ := diff.Diff(prev_item, parsers[parser].Evt) changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
for _, change := range changelog { for _, change := range changelog {
switch change.Type { switch change.Type {
case "create": case "create":
created++ created++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To)) detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
case "update": case "update":
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To)) detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
if change.Path[0] == "Whitelisted" && change.To == true { if change.Path[0] == "Whitelisted" && change.To == true {
whitelisted = true whitelisted = true
if whitelistReason == "" { if whitelistReason == "" {
whitelistReason = parsers[parser].Evt.WhitelistReason whitelistReason = parsers[parser].Evt.WhitelistReason
} }
@ -468,51 +515,64 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
updated++ updated++
case "delete": case "delete":
deleted++ deleted++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, "."))) detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
} }
} }
prev_item = parsers[parser].Evt
prevItem = parsers[parser].Evt
} }
if created > 0 { if created > 0 {
changeStr += green(fmt.Sprintf("+%d", created)) changeStr += green(fmt.Sprintf("+%d", created))
} }
if updated > 0 { if updated > 0 {
if len(changeStr) > 0 { if len(changeStr) > 0 {
changeStr += " " changeStr += " "
} }
changeStr += yellow(fmt.Sprintf("~%d", updated)) changeStr += yellow(fmt.Sprintf("~%d", updated))
} }
if deleted > 0 { if deleted > 0 {
if len(changeStr) > 0 { if len(changeStr) > 0 {
changeStr += " " changeStr += " "
} }
changeStr += red(fmt.Sprintf("-%d", deleted)) changeStr += red(fmt.Sprintf("-%d", deleted))
} }
if whitelisted { if whitelisted {
if len(changeStr) > 0 { if len(changeStr) > 0 {
changeStr += " " changeStr += " "
} }
changeStr += red("[whitelisted]") changeStr += red("[whitelisted]")
} }
if changeStr == "" { if changeStr == "" {
changeStr = yellow("unchanged") changeStr = yellow("unchanged")
} }
if res { if res {
fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr) fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
if opts.Details { if opts.Details {
fmt.Print(detailsDisplay) fmt.Print(detailsDisplay)
} }
} else if opts.ShowNotOkParsers { } else if opts.ShowNotOkParsers {
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser) fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
}
}
}
}
}
}
sep := "└" sep := "└"
if len(state[tstamp]["buckets"]) > 0 { if len(state[tstamp]["buckets"]) > 0 {
sep = "├" sep = "├"
} }
//did the event enter the bucket pour phase ? //did the event enter the bucket pour phase ?
if _, ok := state[tstamp]["buckets"]["OK"]; ok { if _, ok := state[tstamp]["buckets"]["OK"]; ok {
fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle) fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
@ -521,27 +581,35 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
} else { } else {
fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle) fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
} }
//now print bucket info //now print bucket info
if len(state[tstamp]["buckets"]) > 0 { if len(state[tstamp]["buckets"]) > 0 {
fmt.Printf("\t├ Scenarios\n") fmt.Printf("\t├ Scenarios\n")
} }
bnames := make([]string, 0, len(state[tstamp]["buckets"])) bnames := make([]string, 0, len(state[tstamp]["buckets"]))
for k := range state[tstamp]["buckets"] { for k := range state[tstamp]["buckets"] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase //there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered //we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "OK" { if k == "OK" {
continue continue
} }
bnames = append(bnames, k) bnames = append(bnames, k)
} }
sort.Strings(bnames) sort.Strings(bnames)
for idx, bname := range bnames { for idx, bname := range bnames {
sep := "├" sep := "├"
if idx == len(bnames)-1 { if idx == len(bnames)-1 {
sep = "└" sep = "└"
} }
fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname) fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
} }
fmt.Println() fmt.Println()
} }
} }

11
pkg/hubtest/regexp.go Normal file
View file

@ -0,0 +1,11 @@
package hubtest
import (
"regexp"
)
var (
variableRE = regexp.MustCompile(`(?P<variable>[^ =]+) == .*`)
parserResultRE = regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
scenarioResultRE = regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
)

57
pkg/hubtest/scenario_assert.go
View file

@ -5,12 +5,10 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"regexp"
"sort" "sort"
"strings" "strings"
"github.com/antonmedv/expr" "github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
@ -42,6 +40,7 @@ func NewScenarioAssert(file string) *ScenarioAssert {
TestData: &BucketResults{}, TestData: &BucketResults{},
PourData: &BucketPourInfo{}, PourData: &BucketPourInfo{},
} }
return ScenarioAssert return ScenarioAssert
} }
@ -50,7 +49,9 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
ret := s.AutoGenScenarioAssert() ret := s.AutoGenScenarioAssert()
return ret, nil return ret, nil
} }
@ -59,6 +60,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
if err != nil { if err != nil {
return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err) return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err)
} }
s.TestData = bucketDump s.TestData = bucketDump
if bucketpour != "" { if bucketpour != "" {
@ -66,8 +68,10 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
if err != nil { if err != nil {
return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err) return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
} }
s.PourData = pourDump s.PourData = pourDump
} }
return nil return nil
} }
@ -81,19 +85,26 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
if err := s.LoadTest(testFile, ""); err != nil { if err := s.LoadTest(testFile, ""); err != nil {
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err) return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
} }
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines) scanner.Split(bufio.ScanLines)
nbLine := 0 nbLine := 0
for scanner.Scan() { for scanner.Scan() {
nbLine += 1 nbLine++
if scanner.Text() == "" { if scanner.Text() == "" {
continue continue
} }
ok, err := s.Run(scanner.Text()) ok, err := s.Run(scanner.Text())
if err != nil { if err != nil {
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err) return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
} }
s.NbAssert += 1
s.NbAssert++
if !ok { if !ok {
log.Debugf("%s is FALSE", scanner.Text()) log.Debugf("%s is FALSE", scanner.Text())
failedAssert := &AssertFail{ failedAssert := &AssertFail{
@ -102,31 +113,38 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
Expression: scanner.Text(), Expression: scanner.Text(),
Debug: make(map[string]string), Debug: make(map[string]string),
} }
variableRE := regexp.MustCompile(`(?P<variable>[^ ]+) == .*`)
match := variableRE.FindStringSubmatch(scanner.Text()) match := variableRE.FindStringSubmatch(scanner.Text())
if len(match) == 0 { if len(match) == 0 {
log.Infof("Couldn't get variable of line '%s'", scanner.Text()) log.Infof("Couldn't get variable of line '%s'", scanner.Text())
continue continue
} }
variable := match[1] variable := match[1]
result, err := s.EvalExpression(variable) result, err := s.EvalExpression(variable)
if err != nil { if err != nil {
log.Errorf("unable to evaluate variable '%s': %s", variable, err) log.Errorf("unable to evaluate variable '%s': %s", variable, err)
continue continue
} }
failedAssert.Debug[variable] = result failedAssert.Debug[variable] = result
s.Fails = append(s.Fails, *failedAssert) s.Fails = append(s.Fails, *failedAssert)
continue continue
} }
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text()) //fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
} }
file.Close() file.Close()
if s.NbAssert == 0 { if s.NbAssert == 0 {
assertData, err := s.AutoGenFromFile(testFile) assertData, err := s.AutoGenFromFile(testFile)
if err != nil { if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err) return fmt.Errorf("couldn't generate assertion: %s", err)
} }
s.AutoGenAssertData = assertData s.AutoGenAssertData = assertData
s.AutoGenAssert = true s.AutoGenAssert = true
} }
@ -139,15 +157,14 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
} }
func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
var err error
//debug doesn't make much sense with the ability to evaluate "on the fly" //debug doesn't make much sense with the ability to evaluate "on the fly"
//var debugFilter *exprhelpers.ExprDebugger //var debugFilter *exprhelpers.ExprDebugger
var runtimeFilter *vm.Program
var output interface{} var output interface{}
env := map[string]interface{}{"results": *s.TestData} env := map[string]interface{}{"results": *s.TestData}
if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
if err != nil {
return nil, err return nil, err
} }
// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil {
@ -161,8 +178,10 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
if err != nil { if err != nil {
log.Warningf("running : %s", expression) log.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err) log.Warningf("runtime error : %s", err)
return nil, fmt.Errorf("while running expression %s: %w", expression, err) return nil, fmt.Errorf("while running expression %s: %w", expression, err)
} }
return output, nil return output, nil
} }
@ -171,10 +190,12 @@ func (s *ScenarioAssert) EvalExpression(expression string) (string, error) {
if err != nil { if err != nil {
return "", err return "", err
} }
ret, err := yaml.Marshal(output) ret, err := yaml.Marshal(output)
if err != nil { if err != nil {
return "", err return "", err
} }
return string(ret), nil return string(ret), nil
} }
@ -183,6 +204,7 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
if err != nil { if err != nil {
return false, err return false, err
} }
switch out := output.(type) { switch out := output.(type) {
case bool: case bool:
return out, nil return out, nil
@ -192,9 +214,9 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
} }
func (s *ScenarioAssert) AutoGenScenarioAssert() string { func (s *ScenarioAssert) AutoGenScenarioAssert() string {
//attempt to autogen parser asserts // attempt to autogen scenario asserts
var ret string ret := fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
for eventIndex, event := range *s.TestData { for eventIndex, event := range *s.TestData {
for ipSrc, source := range event.Overflow.Sources { for ipSrc, source := range event.Overflow.Sources {
ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex) ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex)
@ -203,15 +225,18 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string {
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope) ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope)
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value) ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value)
} }
for evtIndex, evt := range event.Overflow.Alert.Events { for evtIndex, evt := range event.Overflow.Alert.Events {
for _, meta := range evt.Meta { for _, meta := range evt.Meta {
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value)) ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value))
} }
} }
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario) ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario)
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation) ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation)
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount) ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount)
} }
return ret return ret
} }
@ -228,8 +253,6 @@ func (b BucketResults) Swap(i, j int) {
} }
func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) { func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
var bucketDump BucketPourInfo
dumpData, err := os.Open(filepath) dumpData, err := os.Open(filepath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -241,6 +264,8 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
return nil, err return nil, err
} }
var bucketDump BucketPourInfo
if err := yaml.Unmarshal(results, &bucketDump); err != nil { if err := yaml.Unmarshal(results, &bucketDump); err != nil {
return nil, err return nil, err
} }
@ -249,8 +274,6 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
} }
func LoadScenarioDump(filepath string) (*BucketResults, error) { func LoadScenarioDump(filepath string) (*BucketResults, error) {
var bucketDump BucketResults
dumpData, err := os.Open(filepath) dumpData, err := os.Open(filepath)
if err != nil { if err != nil {
return nil, err return nil, err
@ -262,6 +285,8 @@ func LoadScenarioDump(filepath string) (*BucketResults, error) {
return nil, err return nil, err
} }
var bucketDump BucketResults
if err := yaml.Unmarshal(results, &bucketDump); err != nil { if err := yaml.Unmarshal(results, &bucketDump); err != nil {
return nil, err return nil, err
} }

8
pkg/hubtest/utils.go
View file

@ -12,7 +12,9 @@ func sortedMapKeys[V any](m map[string]V) []string {
for k := range m { for k := range m {
keys = append(keys, k) keys = append(keys, k)
} }
sort.Strings(keys) sort.Strings(keys)
return keys return keys
} }
@ -22,7 +24,7 @@ func Copy(src string, dst string) error {
return err return err
} }
err = os.WriteFile(dst, content, 0644) err = os.WriteFile(dst, content, 0o644)
if err != nil { if err != nil {
return err return err
} }
@ -43,16 +45,20 @@ func checkPathNotContained(path string, subpath string) error {
} }
current := absSubPath current := absSubPath
for { for {
if current == absPath { if current == absPath {
return fmt.Errorf("cannot copy a folder onto itself") return fmt.Errorf("cannot copy a folder onto itself")
} }
up := filepath.Dir(current) up := filepath.Dir(current)
if current == up { if current == up {
break break
} }
current = up current = up
} }
return nil return nil
} }

18
pkg/hubtest/utils_test.go
View file

@ -3,16 +3,16 @@ package hubtest
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/require"
) )
func TestCheckPathNotContained(t *testing.T) { func TestCheckPathNotContained(t *testing.T) {
assert.Nil(t, checkPathNotContained("/foo", "/bar")) require.NoError(t, checkPathNotContained("/foo", "/bar"))
assert.Nil(t, checkPathNotContained("/foo/bar", "/foo")) require.NoError(t, checkPathNotContained("/foo/bar", "/foo"))
assert.Nil(t, checkPathNotContained("/foo/bar", "/")) require.NoError(t, checkPathNotContained("/foo/bar", "/"))
assert.Nil(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else")) require.NoError(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else"))
assert.Nil(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else")) require.NoError(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else"))
assert.NotNil(t, checkPathNotContained("/foo", "/foo/bar")) require.Error(t, checkPathNotContained("/foo", "/foo/bar"))
assert.NotNil(t, checkPathNotContained("/", "/foo")) require.Error(t, checkPathNotContained("/", "/foo"))
assert.NotNil(t, checkPathNotContained("/", "/foo/bar/baz")) require.Error(t, checkPathNotContained("/", "/foo/bar/baz"))
} }

39
pkg/leakybucket/buckets_test.go
View file

@ -8,12 +8,14 @@ import (
"html/template" "html/template"
"io" "io"
"os" "os"
"path/filepath"
"reflect" "reflect"
"sync" "sync"
"testing" "testing"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
@ -33,28 +35,45 @@ func TestBucket(t *testing.T) {
envSetting = os.Getenv("TEST_ONLY") envSetting = os.Getenv("TEST_ONLY")
tomb = &tomb.Tomb{} tomb = &tomb.Tomb{}
) )
err := exprhelpers.Init(nil)
testdata := "./tests"
hubCfg := &csconfig.LocalHubCfg{
HubDir: filepath.Join(testdata, "hub"),
HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
InstallDataDir: testdata,
}
hub, err := cwhub.NewHub(hubCfg, nil, false)
if err != nil {
t.Fatalf("failed to init hub: %s", err)
}
err = exprhelpers.Init(nil)
if err != nil { if err != nil {
log.Fatalf("exprhelpers init failed: %s", err) log.Fatalf("exprhelpers init failed: %s", err)
} }
if envSetting != "" { if envSetting != "" {
if err := testOneBucket(t, envSetting, tomb); err != nil { if err := testOneBucket(t, hub, envSetting, tomb); err != nil {
t.Fatalf("Test '%s' failed : %s", envSetting, err) t.Fatalf("Test '%s' failed : %s", envSetting, err)
} }
} else { } else {
wg := new(sync.WaitGroup) wg := new(sync.WaitGroup)
fds, err := os.ReadDir("./tests/") fds, err := os.ReadDir(testdata)
if err != nil { if err != nil {
t.Fatalf("Unable to read test directory : %s", err) t.Fatalf("Unable to read test directory : %s", err)
} }
for _, fd := range fds { for _, fd := range fds {
fname := "./tests/" + fd.Name() if fd.Name() == "hub" {
continue
}
fname := filepath.Join(testdata, fd.Name())
log.Infof("Running test on %s", fname) log.Infof("Running test on %s", fname)
tomb.Go(func() error { tomb.Go(func() error {
wg.Add(1) wg.Add(1)
defer wg.Done() defer wg.Done()
if err := testOneBucket(t, fname, tomb); err != nil { if err := testOneBucket(t, hub, fname, tomb); err != nil {
t.Fatalf("Test '%s' failed : %s", fname, err) t.Fatalf("Test '%s' failed : %s", fname, err)
} }
return nil return nil
@ -76,7 +95,7 @@ func watchTomb(tomb *tomb.Tomb) {
} }
} }
func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) error {
var ( var (
holders []BucketFactory holders []BucketFactory
@ -112,10 +131,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
files = append(files, x.Filename) files = append(files, x.Filename)
} }
cscfg := &csconfig.CrowdsecServiceCfg{ cscfg := &csconfig.CrowdsecServiceCfg{}
DataDir: "tests", holders, response, err := LoadBuckets(cscfg, hub, files, tomb, buckets, false)
}
holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false)
if err != nil { if err != nil {
t.Fatalf("failed loading bucket : %s", err) t.Fatalf("failed loading bucket : %s", err)
} }
@ -123,7 +140,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
watchTomb(tomb) watchTomb(tomb)
return nil return nil
}) })
if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) { if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) {
return fmt.Errorf("tests from %s failed", dir) return fmt.Errorf("tests from %s failed", dir)
} }
return nil return nil

10
pkg/leakybucket/manager_load.go
View file

@ -178,7 +178,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
return nil return nil
} }
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
var ( var (
ret = []BucketFactory{} ret = []BucketFactory{}
response chan types.Event response chan types.Event
@ -211,7 +211,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
log.Tracef("End of yaml file") log.Tracef("End of yaml file")
break break
} }
bucketFactory.DataDir = cscfg.DataDir bucketFactory.DataDir = hub.GetDataDir()
//check empty //check empty
if bucketFactory.Name == "" { if bucketFactory.Name == "" {
log.Errorf("Won't load nameless bucket") log.Errorf("Won't load nameless bucket")
@ -234,7 +234,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
bucketFactory.Filename = filepath.Clean(f) bucketFactory.Filename = filepath.Clean(f)
bucketFactory.BucketName = seed.Generate() bucketFactory.BucketName = seed.Generate()
bucketFactory.ret = response bucketFactory.ret = response
hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename) hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
if err != nil { if err != nil {
log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
} else { } else {
@ -242,8 +242,8 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
bucketFactory.Simulated = cscfg.SimulationConfig.IsSimulated(hubItem.Name) bucketFactory.Simulated = cscfg.SimulationConfig.IsSimulated(hubItem.Name)
} }
if hubItem != nil { if hubItem != nil {
bucketFactory.ScenarioVersion = hubItem.LocalVersion bucketFactory.ScenarioVersion = hubItem.State.LocalVersion
bucketFactory.hash = hubItem.LocalHash bucketFactory.hash = hubItem.State.LocalHash
} else { } else {
log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
} }

1
pkg/leakybucket/tests/hub/index.json Normal file
View file

@ -0,0 +1 @@
{}

21
pkg/parser/unix_parser.go
View file

@ -57,24 +57,25 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
// Return new parsers // Return new parsers
// nodes and povfwnodes are already initialized in parser.LoadStages // nodes and povfwnodes are already initialized in parser.LoadStages
func NewParsers() *Parsers { func NewParsers(hub *cwhub.Hub) *Parsers {
parsers := &Parsers{ parsers := &Parsers{
Ctx: &UnixParserCtx{}, Ctx: &UnixParserCtx{},
Povfwctx: &UnixParserCtx{}, Povfwctx: &UnixParserCtx{},
StageFiles: make([]Stagefile, 0), StageFiles: make([]Stagefile, 0),
PovfwStageFiles: make([]Stagefile, 0), PovfwStageFiles: make([]Stagefile, 0),
} }
for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
for _, hubParserItem := range cwhub.GetItemMap(itemType) { for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
if hubParserItem.Installed { for _, hubParserItem := range hub.GetItemMap(itemType) {
if hubParserItem.State.Installed {
stagefile := Stagefile{ stagefile := Stagefile{
Filename: hubParserItem.LocalPath, Filename: hubParserItem.State.LocalPath,
Stage: hubParserItem.Stage, Stage: hubParserItem.Stage,
} }
if itemType == cwhub.PARSERS { if itemType == cwhub.PARSERS {
parsers.StageFiles = append(parsers.StageFiles, stagefile) parsers.StageFiles = append(parsers.StageFiles, stagefile)
} }
if itemType == cwhub.PARSERS_OVFLW { if itemType == cwhub.POSTOVERFLOWS {
parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile) parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
} }
} }
@ -97,16 +98,16 @@ func NewParsers() *Parsers {
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error var err error
patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/") patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/")
log.Infof("Loading grok library %s", patternsDir) log.Infof("Loading grok library %s", patternsDir)
/* load base regexps for two grok parsers */ /* load base regexps for two grok parsers */
parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir, parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.Crowdsec.DataDir}) "data": cConfig.ConfigPaths.DataDir})
if err != nil { if err != nil {
return parsers, fmt.Errorf("failed to load parser patterns : %v", err) return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
} }
parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir, parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.Crowdsec.DataDir}) "data": cConfig.ConfigPaths.DataDir})
if err != nil { if err != nil {
return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err) return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
} }
@ -116,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
*/ */
log.Infof("Loading enrich plugins") log.Infof("Loading enrich plugins")
parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
if err != nil { if err != nil {
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
} }

49
pkg/setup/install.go
View file

@ -10,7 +10,6 @@ import (
goccyyaml "github.com/goccy/go-yaml" goccyyaml "github.com/goccy/go-yaml"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
) )
@ -46,22 +45,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
} }
// InstallHubItems installs the objects recommended in a setup file. // InstallHubItems installs the objects recommended in a setup file.
func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error { func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
setupEnvelope, err := decodeSetup(input, false) setupEnvelope, err := decodeSetup(input, false)
if err != nil { if err != nil {
return err return err
} }
if err := csConfig.LoadHub(); err != nil {
return fmt.Errorf("loading hub: %w", err)
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("getting hub index: %w", err)
}
for _, setupItem := range setupEnvelope.Setup { for _, setupItem := range setupEnvelope.Setup {
forceAction := false forceAction := false
downloadOnly := false downloadOnly := false
@ -73,14 +62,19 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
if len(install.Collections) > 0 { if len(install.Collections) > 0 {
for _, collection := range setupItem.Install.Collections { for _, collection := range setupItem.Install.Collections {
item := hub.GetItem(cwhub.COLLECTIONS, collection)
if item == nil {
return fmt.Errorf("collection %s not found", collection)
}
if dryRun { if dryRun {
fmt.Println("dry-run: would install collection", collection) fmt.Println("dry-run: would install collection", collection)
continue continue
} }
if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", collection, err) return fmt.Errorf("while installing collection %s: %w", item.Name, err)
} }
} }
} }
@ -93,8 +87,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue continue
} }
if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { item := hub.GetItem(cwhub.PARSERS, parser)
return fmt.Errorf("while installing parser %s: %w", parser, err) if item == nil {
return fmt.Errorf("parser %s not found", parser)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", item.Name, err)
} }
} }
} }
@ -107,8 +106,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue continue
} }
if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { item := hub.GetItem(cwhub.SCENARIOS, scenario)
return fmt.Errorf("while installing scenario %s: %w", scenario, err) if item == nil {
return fmt.Errorf("scenario %s not found", scenario)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", item.Name, err)
} }
} }
} }
@ -121,8 +125,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue continue
} }
if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow)
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) if item == nil {
return fmt.Errorf("postoverflow %s not found", postoverflow)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err)
} }
} }
} }

71
test/bats/00_wait_for.bats Normal file
View file

@ -0,0 +1,71 @@
#!/usr/bin/env bats
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
set -u
setup_file() {
load "../lib/setup_file.sh"
}
setup() {
load "../lib/setup.sh"
}
@test "run a command and capture its stdout" {
run -0 wait-for seq 1 3
assert_output - <<-EOT
1
2
3
EOT
}
@test "run a command and capture its stderr" {
rune -0 wait-for sh -c 'seq 1 3 >&2'
assert_stderr - <<-EOT
1
2
3
EOT
}
@test "run a command until a pattern is found in stdout" {
run -0 wait-for --out "1[12]0" seq 1 200
assert_line --index 0 "1"
assert_line --index -1 "110"
refute_line "111"
}
@test "run a command until a pattern is found in stderr" {
rune -0 wait-for --err "10" sh -c 'seq 1 20 >&2'
assert_stderr - <<-EOT
1
2
3
4
5
6
7
8
9
10
EOT
}
@test "run a command with timeout (no match)" {
# when the process is terminated without a match, it returns
# 256 - 15 (SIGTERM) = 241
rune -241 wait-for --timeout 0.1 --out "10" sh -c 'echo 1; sleep 3; echo 2'
assert_line 1
# there may be more, but we don't care
}
@test "run a command with timeout (match)" {
# when the process is terminated with a match, return code is 128
rune -128 wait-for --timeout .4 --out "2" sh -c 'echo 1; sleep .1; echo 2; echo 3; echo 4; sleep 10'
assert_output - <<-EOT
1
2
EOT
}

57
test/bats/01_crowdsec.bats
View file

@ -24,28 +24,22 @@ teardown() {
#---------- #----------
@test "crowdsec (usage)" { @test "crowdsec (usage)" {
rune -0 timeout 2s "${CROWDSEC}" -h rune -0 wait-for --out "Usage of " "${CROWDSEC}" -h
assert_stderr_line --regexp "Usage of .*:" rune -0 wait-for --out "Usage of " "${CROWDSEC}" --help
rune -0 timeout 2s "${CROWDSEC}" --help
assert_stderr_line --regexp "Usage of .*:"
} }
@test "crowdsec (unknown flag)" { @test "crowdsec (unknown flag)" {
rune -2 timeout 2s "${CROWDSEC}" --foobar rune -0 wait-for --err "flag provided but not defined: -foobar" "$CROWDSEC" --foobar
assert_stderr_line "flag provided but not defined: -foobar"
assert_stderr_line --regexp "Usage of .*"
} }
@test "crowdsec (unknown argument)" { @test "crowdsec (unknown argument)" {
rune -2 timeout 2s "${CROWDSEC}" trololo rune -0 wait-for --err "argument provided but not defined: trololo" "${CROWDSEC}" trololo
assert_stderr_line "argument provided but not defined: trololo"
assert_stderr_line --regexp "Usage of .*"
} }
@test "crowdsec (no api and no agent)" { @test "crowdsec (no api and no agent)" {
rune -1 timeout 2s "${CROWDSEC}" -no-api -no-cs rune -0 wait-for \
assert_stderr_line --partial "You must run at least the API Server or crowdsec" --err "You must run at least the API Server or crowdsec" \
"${CROWDSEC}" -no-api -no-cs
} }
@test "crowdsec - print error on exit" { @test "crowdsec - print error on exit" {
@ -55,20 +49,22 @@ teardown() {
assert_stderr --partial "unable to create database client: unknown database type 'meh'" assert_stderr --partial "unable to create database client: unknown database type 'meh'"
} }
@test "crowdsec - bad configuration (empty/missing common section)" { @test "crowdsec - default logging configuration (empty/missing common section)" {
config_set '.common={}' config_set '.common={}'
rune -1 "${CROWDSEC}" rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
refute_output refute_output
assert_stderr --partial "unable to load configuration: common section is empty"
config_set 'del(.common)' config_set 'del(.common)'
rune -1 "${CROWDSEC}" rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
refute_output refute_output
assert_stderr --partial "unable to load configuration: common section is empty"
} }
@test "CS_LAPI_SECRET not strong enough" { @test "CS_LAPI_SECRET not strong enough" {
CS_LAPI_SECRET=foo rune -1 timeout 2s "${CROWDSEC}" CS_LAPI_SECRET=foo rune -1 wait-for "${CROWDSEC}"
assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough" assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
} }
@ -138,8 +134,8 @@ teardown() {
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
rm -f "$ACQUIS_YAML" rm -f "$ACQUIS_YAML"
rune -1 timeout 2s "${CROWDSEC}" rune -1 wait-for "${CROWDSEC}"
assert_stderr_line --partial "acquis.yaml: no such file or directory" assert_stderr --partial "acquis.yaml: no such file or directory"
} }
@test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" { @test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" {
@ -151,7 +147,7 @@ teardown() {
rm -f "$ACQUIS_DIR" rm -f "$ACQUIS_DIR"
config_set '.common.log_media="stdout"' config_set '.common.log_media="stdout"'
rune -1 timeout 2s "${CROWDSEC}" rune -1 wait-for "${CROWDSEC}"
# check warning # check warning
assert_stderr --partial "no acquisition file found" assert_stderr --partial "no acquisition file found"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
@ -167,13 +163,15 @@ teardown() {
config_set '.crowdsec_service.acquisition_dir=""' config_set '.crowdsec_service.acquisition_dir=""'
config_set '.common.log_media="stdout"' config_set '.common.log_media="stdout"'
rune -1 timeout 2s "${CROWDSEC}" rune -1 wait-for "${CROWDSEC}"
# check warning # check warning
assert_stderr --partial "no acquisition_path or acquisition_dir specified" assert_stderr --partial "no acquisition_path or acquisition_dir specified"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
} }
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
config_set '.common.log_media="stdout"'
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
config_set '.crowdsec_service.acquisition_path=""' config_set '.crowdsec_service.acquisition_path=""'
@ -181,13 +179,15 @@ teardown() {
mkdir -p "$ACQUIS_DIR" mkdir -p "$ACQUIS_DIR"
mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
rune -124 timeout 2s "${CROWDSEC}" rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
# now, if foo.yaml is empty instead, there won't be valid datasources. # now, if foo.yaml is empty instead, there won't be valid datasources.
cat /dev/null >"$ACQUIS_DIR"/foo.yaml cat /dev/null >"$ACQUIS_DIR"/foo.yaml
rune -1 timeout 2s "${CROWDSEC}" rune -1 wait-for "${CROWDSEC}"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
} }
@ -212,9 +212,10 @@ teardown() {
type: syslog type: syslog
EOT EOT
rune -124 timeout 2s env PATH='' "${CROWDSEC}"
#shellcheck disable=SC2016 #shellcheck disable=SC2016
assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' rune -0 wait-for \
--err 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in ' \
env PATH='' "${CROWDSEC}"
# if all datasources are disabled, crowdsec should exit # if all datasources are disabled, crowdsec should exit
@ -222,7 +223,7 @@ teardown() {
rm -f "$ACQUIS_YAML" rm -f "$ACQUIS_YAML"
config_set '.crowdsec_service.acquisition_path=""' config_set '.crowdsec_service.acquisition_path=""'
rune -1 timeout 2s env PATH='' "${CROWDSEC}" rune -1 wait-for env PATH='' "${CROWDSEC}"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
} }

78
test/bats/01_cscli.bats
View file

@ -110,6 +110,37 @@ teardown() {
assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]' assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]'
} }
@test "cscli - required configuration paths" {
config=$(cat "${CONFIG_YAML}")
configdir=$(config_get '.config_paths.config_dir')
# required configuration paths with no defaults
config_set 'del(.config_paths)'
rune -1 cscli hub list
assert_stderr --partial 'no configuration paths provided'
echo "$config" > "${CONFIG_YAML}"
config_set 'del(.config_paths.data_dir)'
rune -1 cscli hub list
assert_stderr --partial "please provide a data directory with the 'data_dir' directive in the 'config_paths' section"
echo "$config" > "${CONFIG_YAML}"
# defaults
config_set 'del(.config_paths.hub_dir)'
rune -0 cscli hub list
rune -0 cscli config show --key Config.ConfigPaths.HubDir
assert_output "$configdir/hub"
echo "$config" > "${CONFIG_YAML}"
config_set 'del(.config_paths.index_path)'
rune -0 cscli hub list
rune -0 cscli config show --key Config.ConfigPaths.HubIndexFile
assert_output "$configdir/hub/.index.json"
echo "$config" > "${CONFIG_YAML}"
}
@test "cscli config show-yaml" { @test "cscli config show-yaml" {
rune -0 cscli config show-yaml rune -0 cscli config show-yaml
rune -0 yq .common.log_level <(output) rune -0 yq .common.log_level <(output)
@ -245,50 +276,23 @@ teardown() {
assert_output --partial "# bash completion for cscli" assert_output --partial "# bash completion for cscli"
} }
@test "cscli hub list" {
# we check for the presence of some objects. There may be others when we
# use $PACKAGE_TESTING, so the order is not important.
rune -0 cscli hub list -o human
assert_line --regexp '^ crowdsecurity/linux'
assert_line --regexp '^ crowdsecurity/sshd'
assert_line --regexp '^ crowdsecurity/dateparse-enrich'
assert_line --regexp '^ crowdsecurity/geoip-enrich'
assert_line --regexp '^ crowdsecurity/sshd-logs'
assert_line --regexp '^ crowdsecurity/syslog-logs'
assert_line --regexp '^ crowdsecurity/ssh-bf'
assert_line --regexp '^ crowdsecurity/ssh-slow-bf'
rune -0 cscli hub list -o raw
assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$'
assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$'
assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$'
assert_line --regexp '^crowdsecurity/geoip-enrich,enabled,[0-9]+\.[0-9]+,"Populate event with geoloc info : as, country, coords, source range.",parsers$'
assert_line --regexp '^crowdsecurity/sshd-logs,enabled,[0-9]+\.[0-9]+,Parse openSSH logs,parsers$'
assert_line --regexp '^crowdsecurity/syslog-logs,enabled,[0-9]+\.[0-9]+,,parsers$'
assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$'
assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$'
rune -0 cscli hub list -o json
rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name' <(output)
assert_line 'crowdsecurity/linux'
assert_line 'crowdsecurity/sshd'
assert_line 'crowdsecurity/dateparse-enrich'
assert_line 'crowdsecurity/geoip-enrich'
assert_line 'crowdsecurity/sshd-logs'
assert_line 'crowdsecurity/syslog-logs'
assert_line 'crowdsecurity/ssh-bf'
assert_line 'crowdsecurity/ssh-slow-bf'
}
@test "cscli support dump (smoke test)" { @test "cscli support dump (smoke test)" {
rune -0 cscli support dump -f "$BATS_TEST_TMPDIR"/dump.zip rune -0 cscli support dump -f "$BATS_TEST_TMPDIR"/dump.zip
assert_file_exists "$BATS_TEST_TMPDIR"/dump.zip assert_file_exists "$BATS_TEST_TMPDIR"/dump.zip
} }
@test "cscli explain" { @test "cscli explain" {
rune -0 cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog --crowdsec "$CROWDSEC" line="Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4"
rune -0 cscli parsers install crowdsecurity/syslog-logs
rune -0 cscli collections install crowdsecurity/sshd
rune -0 cscli explain --log "$line" --type syslog --only-successful-parsers --crowdsec "$CROWDSEC"
assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt
rune -0 cscli parsers remove --all --purge
rune -1 cscli explain --log "$line" --type syslog --crowdsec "$CROWDSEC"
assert_stderr --partial "unable to load parser dump result: no parser found. Please install the appropriate parser and retry"
} }
@test 'Allow variable expansion and literal $ characters in passwords' { @test 'Allow variable expansion and literal $ characters in passwords' {

16
test/bats/02_nolapi.bats
View file

@ -24,21 +24,23 @@ teardown() {
#---------- #----------
@test "test without -no-api flag" { @test "test without -no-api flag" {
rune -124 timeout 2s "${CROWDSEC}" config_set '.common.log_media="stdout"'
# from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. rune -0 wait-for \
--err "CrowdSec Local API listening" \
"${CROWDSEC}"
} }
@test "crowdsec should not run without LAPI (-no-api flag)" { @test "crowdsec should not run without LAPI (-no-api flag)" {
# really needs 4 secs on slow boxes config_set '.common.log_media="stdout"'
rune -1 timeout 4s "${CROWDSEC}" -no-api rune -1 wait-for "${CROWDSEC}" -no-api
} }
@test "crowdsec should not run without LAPI (no api.server in configuration file)" { @test "crowdsec should not run without LAPI (no api.server in configuration file)" {
config_disable_lapi config_disable_lapi
config_log_stderr config_log_stderr
# really needs 4 secs on slow boxes rune -0 wait-for \
rune -1 timeout 4s "${CROWDSEC}" --err "crowdsec local API is disabled" \
assert_stderr --partial "crowdsec local API is disabled" "${CROWDSEC}"
} }
@test "capi status shouldn't be ok without api.server" { @test "capi status shouldn't be ok without api.server" {

17
test/bats/03_noagent.bats
View file

@ -23,20 +23,25 @@ teardown() {
#---------- #----------
@test "with agent: test without -no-cs flag" { @test "with agent: test without -no-cs flag" {
rune -124 timeout 2s "${CROWDSEC}" config_set '.common.log_media="stdout"'
# from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124. rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
} }
@test "no agent: crowdsec LAPI should run (-no-cs flag)" { @test "no agent: crowdsec LAPI should run (-no-cs flag)" {
rune -124 timeout 2s "${CROWDSEC}" -no-cs config_set '.common.log_media="stdout"'
rune -0 wait-for \
--err "CrowdSec Local API listening" \
"${CROWDSEC}" -no-cs
} }
@test "no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" { @test "no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" {
config_disable_agent config_disable_agent
config_log_stderr config_log_stderr
rune -124 timeout 2s "${CROWDSEC}" rune -0 wait-for \
--err "crowdsec agent is disabled" \
assert_stderr --partial "crowdsec agent is disabled" "${CROWDSEC}"
} }
@test "no agent: cscli config show" { @test "no agent: cscli config show" {

4
test/bats/04_capi.bats
View file

@ -22,6 +22,10 @@ setup() {
@test "cscli capi status" { @test "cscli capi status" {
config_enable_capi config_enable_capi
rune -0 cscli capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX rune -0 cscli capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX
rune -1 cscli capi status
assert_stderr --partial "no scenarios installed, abort"
rune -0 cscli scenarios install crowdsecurity/ssh-bf
rune -0 cscli capi status rune -0 cscli capi status
assert_stderr --partial "Loaded credentials from" assert_stderr --partial "Loaded credentials from"
assert_stderr --partial "Trying to authenticate with username" assert_stderr --partial "Trying to authenticate with username"

Some files were not shown because too many files have changed in this diff Show more