beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor hub management and cscli commands (#2545)

Browse source
This commit is contained in:
mmetc 2023-11-24 15:57:32 +01:00 committed by GitHub
parent 32e9eb4be4
commit ffcab0b2bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
124 changed files with 6836 additions and 4414 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmd/crowdsec-cli/capi.go
View file

@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command {
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}

176
cmd/crowdsec-cli/collections.go
View file

@ -1,176 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCollectionsCmd() *cobra.Command {
var cmdCollections = &cobra.Command{
Use: "collections [action]",
Short: "Manage collections from hub",
Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
/*TBD fix help*/
Args: cobra.MinimumNArgs(1),
Aliases: []string{"collection"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
var ignoreError bool
var cmdCollectionsInstall = &cobra.Command{
Use: "install collection",
Short: "Install given collection(s)",
Long: `Fetch and install given collection(s) from hub`,
Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.COLLECTIONS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
cmdCollections.AddCommand(cmdCollectionsInstall)
var cmdCollectionsRemove = &cobra.Command{
Use: "remove collection",
Short: "Remove given collection(s)",
Long: `Remove given collection(s) from hub`,
Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one collection to remove or '--all'")
}
for _, name := range args {
if !forceAction {
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
if item == nil {
return fmt.Errorf("unable to retrieve: %s", name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
continue
}
}
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
}
return nil
},
}
cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
cmdCollections.AddCommand(cmdCollectionsRemove)
var cmdCollectionsUpgrade = &cobra.Command{
Use: "upgrade collection",
Short: "Upgrade given collection(s)",
Long: `Fetch and upgrade given collection(s) from hub`,
Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one collection to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
}
}
return nil
},
}
cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
cmdCollections.AddCommand(cmdCollectionsUpgrade)
var cmdCollectionsInspect = &cobra.Command{
Use: "inspect collection",
Short: "Inspect given collection",
Long: `Inspect given collection`,
Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
InspectItem(name, cwhub.COLLECTIONS)
}
},
}
cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
cmdCollections.AddCommand(cmdCollectionsInspect)
var cmdCollectionsList = &cobra.Command{
Use: "list collection [-a]",
Short: "List all collections",
Long: `List all collections`,
Example: `cscli collections list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
},
}
cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
cmdCollections.AddCommand(cmdCollectionsList)
return cmdCollections
}

38
cmd/crowdsec-cli/config_backup.go
View file

@ -14,21 +14,25 @@ import (
)
func backupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
itemMap := hub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
@ -36,30 +40,30 @@ func backupHub(dirPath string) error {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
if !v.State.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we back up the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to back up stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
if v.State.Tainted || v.IsLocal() || !v.State.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.IsLocal(), v.State.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
if err = CopyFile(v.State.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
@ -100,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error {
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath)
if _, err := os.Stat(parentDir); err != nil {
if _, err = os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
}
@ -197,10 +201,6 @@ func backupConfigToDirectory(dirPath string) error {
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
if err := backupConfigToDirectory(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}

54
cmd/crowdsec-cli/config_restore.go
View file

@ -21,45 +21,12 @@ type OldAPICfg struct {
Password string `json:"password"`
}
// it's a rip of the cli version, but in silent-mode
func silentInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
if downloadOnly && item.Downloaded && item.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
}
err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
}
err = cwhub.EnableItem(csConfig.Hub, item)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", item.Name), nil
}
func restoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
cwhub.SetHubBranch()
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
@ -78,13 +45,14 @@ func restoreHub(dirPath string) error {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silentInstallItem(toinstall, itype)
item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
err := item.Install(false, false)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
@ -98,7 +66,7 @@ func restoreHub(dirPath string) error {
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
//we expect a stage here
if !file.IsDir() {
continue
@ -302,10 +270,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return err
}
if err := require.Hub(csConfig); err != nil {
return err
}
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
}

1
cmd/crowdsec-cli/config_show.go
View file

@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
cscli:
- Output : {{.Cscli.Output}}
- Hub Branch : {{.Cscli.HubBranch}}
- Hub Folder : {{.Cscli.HubDir}}
{{- end }}
{{- if .API }}

5
cmd/crowdsec-cli/console.go
View file

@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba
return fmt.Errorf("could not parse CAPI URL: %s", err)
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get installed scenarios: %s", err)
}

208
cmd/crowdsec-cli/hub.go
View file

@ -1,7 +1,6 @@
package main
import (
"errors"
"fmt"
"github.com/fatih/color"
@ -13,31 +12,20 @@ import (
)
func NewHubCmd() *cobra.Command {
var cmdHub = &cobra.Command{
cmdHub := &cobra.Command{
Use: "hub [action]",
Short: "Manage Hub",
Long: `
Hub management
Short: "Manage hub index",
Long: `Hub management
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
`,
Example: `
cscli hub list # List all installed configurations
cscli hub update # Download list of available configurations from the hub
`,
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
Example: `cscli hub list
cscli hub update
cscli hub upgrade`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
return nil
},
}
cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd())
cmdHub.AddCommand(NewHubUpgradeCmd())
@ -45,116 +33,142 @@ cscli hub update # Download list of available configurations from the hub
return cmdHub
}
func NewHubListCmd() *cobra.Command {
var cmdHubList = &cobra.Command{
Use: "list [-a]",
Short: "List installed configs",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
func runHubList(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
if err != nil {
return err
}
// use LocalSync to get warnings about tainted / outdated items
warn, _ := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, v := range hub.Warnings {
log.Info(v)
}
cwhub.DisplaySummary()
ListItems(color.Output, []string{
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
}, args, true, false, all)
for _, line := range hub.ItemStats() {
log.Info(line)
}
items := make(map[string][]*cwhub.Item)
for _, itemType := range cwhub.ItemTypes {
items[itemType], err = selectItems(hub, itemType, nil, !all)
if err != nil {
return err
}
}
err = listItems(color.Output, cwhub.ItemTypes, items)
if err != nil {
return err
}
return nil
},
}
func NewHubListCmd() *cobra.Command {
cmdHubList := &cobra.Command{
Use: "list [-a]",
Short: "List all installed configurations",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubList,
}
cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
flags := cmdHubList.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
return cmdHubList
}
func NewHubUpdateCmd() *cobra.Command {
var cmdHubUpdate = &cobra.Command{
Use: "update",
Short: "Fetch available configs from hub",
Long: `
Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
func runHubUpdate(cmd *cobra.Command, args []string) error {
local := csConfig.Hub
remote := require.RemoteHub(csConfig)
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.NewHub(local, remote, true)
if err != nil {
return fmt.Errorf("failed to update hub: %w", err)
}
cwhub.SetHubBranch()
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)
}
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("failed to get Hub index after retry: %w", err)
}
}
// use LocalSync to get warnings about tainted / outdated items
warn, _ := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
for _, v := range hub.Warnings {
log.Info(v)
}
return nil
},
}
func NewHubUpdateCmd() *cobra.Command {
cmdHubUpdate := &cobra.Command{
Use: "update",
Short: "Download the latest index (catalog of available configurations)",
Long: `
Fetches the .index.json file from the hub, containing the list of available configs.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubUpdate,
}
return cmdHubUpdate
}
func runHubUpgrade(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
items, err := hub.GetInstalledItems(itemType)
if err != nil {
return err
}
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}
return nil
}
func NewHubUpgradeCmd() *cobra.Command {
var cmdHubUpgrade = &cobra.Command{
cmdHubUpgrade := &cobra.Command{
Use: "upgrade",
Short: "Upgrade all configs installed from hub",
Short: "Upgrade all configurations to their latest version",
Long: `
Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
RunE: runHubUpgrade,
}
cwhub.SetHubBranch()
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
log.Infof("Upgrading collections")
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
log.Infof("Upgrading parsers")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
log.Infof("Upgrading scenarios")
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
log.Infof("Upgrading postoverflows")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
return nil
},
}
cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
flags := cmdHubUpgrade.Flags()
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return cmdHubUpgrade
}

66
cmd/crowdsec-cli/hubtest.go
View file

@ -18,9 +18,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
var (
HubTest hubtest.HubTest
)
var HubTest hubtest.HubTest
func NewHubTestCmd() *cobra.Command {
var hubPath string
@ -43,6 +41,7 @@ func NewHubTestCmd() *cobra.Command {
return nil
},
}
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
@ -59,7 +58,6 @@ func NewHubTestCmd() *cobra.Command {
return cmdHubTest
}
func NewHubTestCreateCmd() *cobra.Command {
parsers := []string{}
postoverflows := []string{}
@ -138,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
}
configFilePath := filepath.Join(testPath, "config.yaml")
fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
fd, err := os.Create(configFilePath)
if err != nil {
return fmt.Errorf("open: %s", err)
}
@ -164,6 +162,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return nil
},
}
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
@ -173,7 +172,6 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return cmdHubTestCreate
}
func NewHubTestRunCmd() *cobra.Command {
var noClean bool
var runAll bool
@ -186,7 +184,7 @@ func NewHubTestRunCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) error {
if !runAll && len(args) == 0 {
printHelp(cmd)
return fmt.Errorf("Please provide test to run or --all flag")
return fmt.Errorf("please provide test to run or --all flag")
}
if runAll {
@ -202,6 +200,9 @@ func NewHubTestRunCmd() *cobra.Command {
}
}
// set timezone to avoid DST issues
os.Setenv("TZ", "UTC")
for _, test := range HubTest.Tests {
if csConfig.Cscli.Output == "human" {
log.Infof("Running test '%s'", test.Name)
@ -293,9 +294,11 @@ func NewHubTestRunCmd() *cobra.Command {
}
}
}
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
hubTestResultTable(color.Output, testResult)
} else if csConfig.Cscli.Output == "json" {
case "json":
jsonResult := make(map[string][]string, 0)
jsonResult["success"] = make([]string, 0)
jsonResult["fail"] = make([]string, 0)
@ -311,6 +314,8 @@ func NewHubTestRunCmd() *cobra.Command {
return fmt.Errorf("unable to json test result: %s", err)
}
fmt.Println(string(jsonStr))
default:
return fmt.Errorf("only human/json output modes are supported")
}
if !success {
@ -320,6 +325,7 @@ func NewHubTestRunCmd() *cobra.Command {
return nil
},
}
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
@ -327,7 +333,6 @@ func NewHubTestRunCmd() *cobra.Command {
return cmdHubTestRun
}
func NewHubTestCleanCmd() *cobra.Command {
var cmdHubTestClean = &cobra.Command{
Use: "clean",
@ -352,7 +357,6 @@ func NewHubTestCleanCmd() *cobra.Command {
return cmdHubTestClean
}
func NewHubTestInfoCmd() *cobra.Command {
var cmdHubTestInfo = &cobra.Command{
Use: "info",
@ -381,7 +385,6 @@ func NewHubTestInfoCmd() *cobra.Command {
return cmdHubTestInfo
}
func NewHubTestListCmd() *cobra.Command {
var cmdHubTestList = &cobra.Command{
Use: "list",
@ -412,7 +415,6 @@ func NewHubTestListCmd() *cobra.Command {
return cmdHubTestList
}
func NewHubTestCoverageCmd() *cobra.Command {
var showParserCov bool
var showScenarioCov bool
@ -427,8 +429,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
return fmt.Errorf("unable to load all tests: %+v", err)
}
var err error
scenarioCoverage := []hubtest.ScenarioCoverage{}
parserCoverage := []hubtest.ParserCoverage{}
scenarioCoverage := []hubtest.Coverage{}
parserCoverage := []hubtest.Coverage{}
scenarioCoveragePercent := 0
parserCoveragePercent := 0
@ -443,7 +445,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
parserTested := 0
for _, test := range parserCoverage {
if test.TestsCount > 0 {
parserTested += 1
parserTested++
}
}
parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@ -454,12 +456,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("while getting scenario coverage: %s", err)
}
scenarioTested := 0
for _, test := range scenarioCoverage {
if test.TestsCount > 0 {
scenarioTested += 1
scenarioTested++
}
}
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
}
@ -474,7 +478,8 @@ func NewHubTestCoverageCmd() *cobra.Command {
os.Exit(0)
}
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
if showParserCov || showAll {
hubTestParserCoverageTable(color.Output, parserCoverage)
}
@ -489,7 +494,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
}
} else if csConfig.Cscli.Output == "json" {
case "json":
dump, err := json.MarshalIndent(parserCoverage, "", " ")
if err != nil {
return err
@ -500,13 +505,14 @@ func NewHubTestCoverageCmd() *cobra.Command {
return err
}
fmt.Printf("%s", dump)
} else {
default:
return fmt.Errorf("only human/json output modes are supported")
}
return nil
},
}
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
@ -514,7 +520,6 @@ func NewHubTestCoverageCmd() *cobra.Command {
return cmdHubTestCoverage
}
func NewHubTestEvalCmd() *cobra.Command {
var evalExpression string
var cmdHubTestEval = &cobra.Command{
@ -528,26 +533,29 @@ func NewHubTestEvalCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("can't load test: %+v", err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
}
output, err := test.ParserAssert.EvalExpression(evalExpression)
if err != nil {
return err
}
fmt.Print(output)
}
return nil
},
}
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmdHubTestEval
}
func NewHubTestExplainCmd() *cobra.Command {
var cmdHubTestExplain = &cobra.Command{
Use: "explain",
@ -562,24 +570,22 @@ func NewHubTestExplainCmd() *cobra.Command {
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
err := test.Run()
if err != nil {
if err = test.Run(); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
return fmt.Errorf("unable to load parser result after run: %s", err)
}
}
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
err := test.Run()
if err != nil {
if err = test.Run(); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
return fmt.Errorf("unable to load scenario result after run: %s", err)
}
}

10
cmd/crowdsec-cli/hubtest_table.go
View file

@ -41,39 +41,41 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
t.Render()
}
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Parser", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Scenario", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()

236
cmd/crowdsec-cli/item_metrics.go Normal file
View file

@ -0,0 +1,236 @@
package main
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func ShowMetrics(hubItem *cwhub.Item) error {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, sub := range hubItem.SubItems() {
if err := ShowMetrics(sub); err != nil {
return err
}
}
default:
// no metrics for this item type
}
return nil
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

85
cmd/crowdsec-cli/item_suggest.go Normal file
View file

@ -0,0 +1,85 @@
package main
import (
"fmt"
"strings"
"github.com/agext/levenshtein"
"github.com/spf13/cobra"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
const MaxDistance = 7
// SuggestNearestMessage returns a message with the most similar item name, if one is found
func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
score := 100
nearest := ""
for _, item := range hub.GetItemMap(itemType) {
d := levenshtein.Distance(itemName, item.Name, nil)
if d < score {
score = d
nearest = item.Name
}
}
msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
if score < MaxDistance {
msg += fmt.Sprintf(", did you mean '%s'?", nearest)
}
return msg
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
for _, item := range hub.GetItemMap(itemType) {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := hub.GetInstalledItemNames(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}

606
cmd/crowdsec-cli/itemcommands.go Normal file
View file

@ -0,0 +1,606 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cmdHelp struct {
// Example is required, the others have a default value
// generated from the item type
use string
short string
long string
example string
}
type hubItemType struct {
name string // plural, as used in the hub index
singular string
oneOrMore string // parenthetical pluralizaion: "parser(s)"
help cmdHelp
installHelp cmdHelp
removeHelp cmdHelp
upgradeHelp cmdHelp
inspectHelp cmdHelp
listHelp cmdHelp
}
var hubItemTypes = map[string]hubItemType{
"parsers": {
name: "parsers",
singular: "parser",
oneOrMore: "parser(s)",
help: cmdHelp{
example: `cscli parsers list -a
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
`,
},
installHelp: cmdHelp{
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
removeHelp: cmdHelp{
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
upgradeHelp: cmdHelp{
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
inspectHelp: cmdHelp{
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
},
listHelp: cmdHelp{
example: `cscli parsers list
cscli parsers list -a
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
List only enabled parsers unless "-a" or names are specified.`,
},
},
"postoverflows": {
name: "postoverflows",
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cmdHelp{
example: `cscli postoverflows list -a
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
`,
},
installHelp: cmdHelp{
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
removeHelp: cmdHelp{
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
upgradeHelp: cmdHelp{
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
inspectHelp: cmdHelp{
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
listHelp: cmdHelp{
example: `cscli postoverflows list
cscli postoverflows list -a
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
List only enabled postoverflows unless "-a" or names are specified.`,
},
},
"scenarios": {
name: "scenarios",
singular: "scenario",
oneOrMore: "scenario(s)",
help: cmdHelp{
example: `cscli scenarios list -a
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
`,
},
installHelp: cmdHelp{
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
removeHelp: cmdHelp{
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
upgradeHelp: cmdHelp{
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
inspectHelp: cmdHelp{
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
listHelp: cmdHelp{
example: `cscli scenarios list
cscli scenarios list -a
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
List only enabled scenarios unless "-a" or names are specified.`,
},
},
"collections": {
name: "collections",
singular: "collection",
oneOrMore: "collection(s)",
help: cmdHelp{
example: `cscli collections list -a
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
`,
},
installHelp: cmdHelp{
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
},
removeHelp: cmdHelp{
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
},
upgradeHelp: cmdHelp{
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
},
inspectHelp: cmdHelp{
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
},
listHelp: cmdHelp{
example: `cscli collections list
cscli collections list -a
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
List only enabled collections unless "-a" or names are specified.`,
},
},
}
func NewItemsCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
Long: it.help.long,
Example: it.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{it.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(NewItemsInstallCmd(typeName))
cmd.AddCommand(NewItemsRemoveCmd(typeName))
cmd.AddCommand(NewItemsUpgradeCmd(typeName))
cmd.AddCommand(NewItemsInspectCmd(typeName))
cmd.AddCommand(NewItemsListCmd(typeName))
return cmd
}
func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
downloadOnly, err := flags.GetBool("download-only")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
ignoreError, err := flags.GetBool("ignore")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
msg := SuggestNearestMessage(hub, it.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
log.Errorf(msg)
continue
}
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
log.Infof(ReloadMessage())
return nil
}
return run
}
func NewItemsInstallCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.installHelp.use, "install [item]..."),
Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
Example: it.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(typeName, args, toComplete)
},
RunE: itemsInstallRunner(it),
}
flags := cmd.Flags()
flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
return cmd
}
// return the names of the installed parents of an item, used to check if we can remove it
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.Ancestors() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}
}
return ret
}
func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
purge, err := flags.GetBool("purge")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if all {
getter := hub.GetInstalledItems
if purge {
getter = hub.GetAllItems
}
items, err := getter(it.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
return run
}
func NewItemsRemoveCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.removeHelp.use, "remove [item]..."),
Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
Example: it.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsRemoveRunner(it),
}
flags := cmd.Flags()
flags.Bool("purge", false, "Delete source file too")
flags.Bool("force", false, "Force remove: remove tainted and outdated files")
flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
return cmd
}
func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if all {
items, err := hub.GetInstalledItems(it.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, it.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", item.Name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
return run
}
func NewItemsUpgradeCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
Example: it.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsUpgradeRunner(it),
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
url, err := flags.GetString("url")
if err != nil {
return err
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
noMetrics, err := flags.GetBool("no-metrics")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, it.name)
}
if err = InspectItem(item, !noMetrics); err != nil {
return err
}
}
return nil
}
return run
}
func NewItemsInspectCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
Example: it.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsInspectRunner(it),
}
flags := cmd.Flags()
flags.StringP("url", "u", "", "Prometheus url")
flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
items := make(map[string][]*cwhub.Item)
items[it.name], err = selectItems(hub, it.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{it.name}, items); err != nil {
return err
}
return nil
}
return run
}
func NewItemsListCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
Example: it.listHelp.example,
DisableAutoGenTag: true,
RunE: itemsListRunner(it),
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
return cmd
}

157
cmd/crowdsec-cli/items.go Normal file
View file

@ -0,0 +1,157 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"gopkg.in/yaml.v3"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
itemNames := hub.GetItemNames(itemType)
notExist := []string{}
if len(args) > 0 {
for _, arg := range args {
if !slices.Contains(itemNames, arg) {
notExist = append(notExist, arg)
}
}
}
if len(notExist) > 0 {
return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
}
if len(args) > 0 {
itemNames = args
installedOnly = false
}
items := make([]*cwhub.Item, 0, len(itemNames))
for _, itemName := range itemNames {
item := hub.GetItem(itemType, itemName)
if installedOnly && !item.State.Installed {
continue
}
items = append(items, item)
}
cwhub.SortItemSlice(items)
return items, nil
}
func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item) error {
switch csConfig.Cscli.Output {
case "human":
for _, itemType := range itemTypes {
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
}
case "json":
type itemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
hubStatus := make(map[string][]itemHubStatus)
for _, itemType := range itemTypes {
// empty slice in case there are no items of this type
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
for i, item := range items[itemType] {
status, emo := item.InstallStatus()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.State.LocalVersion,
LocalPath: item.State.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", emo, status),
}
}
}
x, err := json.MarshalIndent(hubStatus, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
out.Write(x)
case "raw":
csvwriter := csv.NewWriter(out)
header := []string{"name", "status", "version", "description"}
if len(itemTypes) > 1 {
header = append(header, "type")
}
if err := csvwriter.Write(header); err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, itemType := range itemTypes {
for _, item := range items[itemType] {
status, _ := item.InstallStatus()
row := []string{
item.Name,
status,
item.State.LocalVersion,
item.Description,
}
if len(itemTypes) > 1 {
row = append(row, itemType)
}
if err := csvwriter.Write(row); err != nil {
return fmt.Errorf("failed to write raw output: %s", err)
}
}
}
csvwriter.Flush()
default:
return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
}
return nil
}
func InspectItem(item *cwhub.Item, showMetrics bool) error {
switch csConfig.Cscli.Output {
case "human", "raw":
enc := yaml.NewEncoder(os.Stdout)
enc.SetIndent(2)
if err := enc.Encode(item); err != nil {
return fmt.Errorf("unable to encode item: %s", err)
}
case "json":
b, err := json.MarshalIndent(*item, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
fmt.Print(string(b))
}
if csConfig.Cscli.Output == "human" && showMetrics {
fmt.Printf("\nCurrent metrics: \n")
if err := ShowMetrics(item); err != nil {
return err
}
}
return nil
}

13
cmd/crowdsec-cli/lapi.go
View file

@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
}
@ -338,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Fatalf("Failed to init expr helpers : %s", err)
}
// Populate cwhub package tools
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
}
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
}

33
cmd/crowdsec-cli/main.go
View file

@ -4,7 +4,6 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"github.com/fatih/color"
@ -12,9 +11,9 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
@ -29,15 +28,11 @@ var dbClient *database.Client
var OutputFormat string
var OutputColor string
var downloadOnly bool
var forceAction bool
var purge bool
var all bool
var prometheusURL string
var mergedConfig string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
var flagBranch = ""
func initConfig() {
var err error
if trace_lvl {
@ -58,9 +53,6 @@ func initConfig() {
if err != nil {
log.Fatal(err)
}
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatal(err)
}
} else {
csConfig = csconfig.NewDefaultConfig()
}
@ -71,13 +63,10 @@ func initConfig() {
log.Debugf("Enabled feature flags: %s", fflist)
}
if csConfig.Cscli == nil {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
if flagBranch != "" {
csConfig.Cscli.HubBranch = flagBranch
}
if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
cwhub.HubBranch = csConfig.Cscli.HubBranch
}
if OutputFormat != "" {
csConfig.Cscli.Output = OutputFormat
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
@ -206,7 +195,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
@ -243,10 +232,6 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(NewSimulationCmds())
rootCmd.AddCommand(NewBouncersCmd())
rootCmd.AddCommand(NewMachinesCmd())
rootCmd.AddCommand(NewParsersCmd())
rootCmd.AddCommand(NewScenariosCmd())
rootCmd.AddCommand(NewCollectionsCmd())
rootCmd.AddCommand(NewPostOverflowsCmd())
rootCmd.AddCommand(NewCapiCmd())
rootCmd.AddCommand(NewLapiCmd())
rootCmd.AddCommand(NewCompletionCmd())
@ -255,6 +240,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(NewHubTestCmd())
rootCmd.AddCommand(NewNotificationsCmd())
rootCmd.AddCommand(NewSupportCmd())
rootCmd.AddCommand(NewItemsCmd("collections"))
rootCmd.AddCommand(NewItemsCmd("parsers"))
rootCmd.AddCommand(NewItemsCmd("scenarios"))
rootCmd.AddCommand(NewItemsCmd("postoverflows"))
if fflag.CscliSetup.IsEnabled() {
rootCmd.AddCommand(NewSetupCmd())

35
cmd/crowdsec-cli/metrics.go
View file

@ -284,8 +284,20 @@ var noUnit bool
func runMetrics(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadPrometheus(); err != nil {
return fmt.Errorf("failed to load prometheus config: %w", err)
flags := cmd.Flags()
url, err := flags.GetString("url")
if err != nil {
return err
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
noUnit, err = flags.GetBool("no-unit")
if err != nil {
return err
}
if csConfig.Prometheus == nil {
@ -296,17 +308,8 @@ func runMetrics(cmd *cobra.Command, args []string) error {
return fmt.Errorf("prometheus is not enabled, can't show metrics")
}
if prometheusURL == "" {
prometheusURL = csConfig.Cscli.PrometheusUrl
}
if prometheusURL == "" {
return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
}
err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
if err != nil {
return fmt.Errorf("could not fetch prometheus metrics: %w", err)
if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
return err
}
return nil
}
@ -321,8 +324,10 @@ func NewMetricsCmd() *cobra.Command {
DisableAutoGenTag: true,
RunE: runMetrics,
}
cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
flags := cmdMetrics.PersistentFlags()
flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
return cmdMetrics
}

194
cmd/crowdsec-cli/parsers.go
View file

@ -1,194 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewParsersCmd() *cobra.Command {
var cmdParsers = &cobra.Command{
Use: "parsers [action] [config]",
Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
Example: `cscli parsers install crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/sshd-logs
cscli parsers list
cscli parsers remove crowdsecurity/sshd-logs
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"parser"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdParsers.AddCommand(NewParsersInstallCmd())
cmdParsers.AddCommand(NewParsersRemoveCmd())
cmdParsers.AddCommand(NewParsersUpgradeCmd())
cmdParsers.AddCommand(NewParsersInspectCmd())
cmdParsers.AddCommand(NewParsersListCmd())
return cmdParsers
}
func NewParsersInstallCmd() *cobra.Command {
var ignoreError bool
var cmdParsersInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given parser(s)",
Long: `Fetch and install given parser(s) from hub`,
Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS, name)
Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
return cmdParsersInstall
}
func NewParsersRemoveCmd() *cobra.Command {
cmdParsersRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given parser(s)",
Long: `Remove given parse(s) from hub`,
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one parser to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
}
return nil
},
}
cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
return cmdParsersRemove
}
func NewParsersUpgradeCmd() *cobra.Command {
cmdParsersUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given parser(s)",
Long: `Fetch and upgrade given parser(s) from hub`,
Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one parser to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
}
}
return nil
},
}
cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdParsersUpgrade
}
func NewParsersInspectCmd() *cobra.Command {
var cmdParsersInspect = &cobra.Command{
Use: "inspect [name]",
Short: "Inspect given parser",
Long: `Inspect given parser`,
Example: `cscli parsers inspect crowdsec/xxx`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS)
},
}
cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdParsersInspect
}
func NewParsersListCmd() *cobra.Command {
var cmdParsersList = &cobra.Command{
Use: "list [name]",
Short: "List all parsers or given one",
Long: `List all parsers or given one`,
Example: `cscli parsers list
cscli parser list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
},
}
cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdParsersList
}

191
cmd/crowdsec-cli/postoverflows.go
View file

@ -1,191 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewPostOverflowsCmd() *cobra.Command {
cmdPostOverflows := &cobra.Command{
Use: "postoverflows [action] [config]",
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
cscli postoverflows inspect crowdsecurity/cdn-whitelist
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
cscli postoverflows list
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
return cmdPostOverflows
}
func NewPostOverflowsInstallCmd() *cobra.Command {
var ignoreError bool
cmdPostOverflowsInstall := &cobra.Command{
Use: "install [config]",
Short: "Install given postoverflow(s)",
Long: `Fetch and install given postoverflow(s) from hub`,
Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
return cmdPostOverflowsInstall
}
func NewPostOverflowsRemoveCmd() *cobra.Command {
cmdPostOverflowsRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given postoverflow(s)",
Long: `remove given postoverflow(s)`,
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
}
return nil
},
}
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
return cmdPostOverflowsRemove
}
func NewPostOverflowsUpgradeCmd() *cobra.Command {
cmdPostOverflowsUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given postoverflow(s)",
Long: `Fetch and Upgrade given postoverflow(s) from hub`,
Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
}
}
return nil
},
}
cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdPostOverflowsUpgrade
}
func NewPostOverflowsInspectCmd() *cobra.Command {
cmdPostOverflowsInspect := &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given postoverflow",
Long: `Inspect given postoverflow`,
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS_OVFLW)
},
}
return cmdPostOverflowsInspect
}
func NewPostOverflowsListCmd() *cobra.Command {
cmdPostOverflowsList := &cobra.Command{
Use: "list [config]",
Short: "List all postoverflows or given one",
Long: `List all postoverflows or given one`,
Example: `cscli postoverflows list
cscli postoverflows list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
},
}
cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdPostOverflowsList
}

58
cmd/crowdsec-cli/require/branch.go Normal file
View file

@ -0,0 +1,58 @@
package require
// Set the appropriate hub branch according to config settings and crowdsec version
import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func chooseBranch(cfg *csconfig.Config) string {
// this was set from config.yaml or flag
if cfg.Cscli.HubBranch != "" {
log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
return cfg.Cscli.HubBranch
}
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using hub branch 'master'")
return "master"
}
log.Warnf("A new CrowdSec release is available (%s). "+
"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
latest, csVersion)
return csVersion
}
// HubBranch sets the branch (in cscli config) and returns its value
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
func HubBranch(cfg *csconfig.Config) string {
branch := chooseBranch(cfg)
cfg.Cscli.HubBranch = branch
return branch
}

44
cmd/crowdsec-cli/require/require.go
View file

@ -23,6 +23,7 @@ func CAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient == nil {
return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
}
return nil
}
@ -30,6 +31,7 @@ func PAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
return fmt.Errorf("no PAPI URL in configuration")
}
return nil
}
@ -45,6 +47,7 @@ func DB(c *csconfig.Config) error {
if err := c.LoadDBConfig(); err != nil {
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
}
return nil
}
@ -64,20 +67,33 @@ func Notifications(c *csconfig.Config) error {
return nil
}
func Hub (c *csconfig.Config) error {
if err := c.LoadHub(); err != nil {
return err
// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// set branch in config, and log if necessary
branch := HubBranch(c)
remote := &cwhub.RemoteHubCfg {
Branch: branch,
URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
IndexPath: ".index.json",
}
if c.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(c.Hub); err != nil {
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
}
return nil
return remote
}
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
local := c.Hub
if local == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub")
}
hub, err := cwhub.NewHub(local, remote, false)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
}
return hub, nil
}

188
cmd/crowdsec-cli/scenarios.go
View file

@ -1,188 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewScenariosCmd() *cobra.Command {
var cmdScenarios = &cobra.Command{
Use: "scenarios [action] [config]",
Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
Example: `cscli scenarios list [-a]
cscli scenarios install crowdsecurity/ssh-bf
cscli scenarios inspect crowdsecurity/ssh-bf
cscli scenarios upgrade crowdsecurity/ssh-bf
cscli scenarios remove crowdsecurity/ssh-bf
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"scenario"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdScenarios.AddCommand(NewCmdScenariosInstall())
cmdScenarios.AddCommand(NewCmdScenariosRemove())
cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
cmdScenarios.AddCommand(NewCmdScenariosInspect())
cmdScenarios.AddCommand(NewCmdScenariosList())
return cmdScenarios
}
func NewCmdScenariosInstall() *cobra.Command {
var ignoreError bool
var cmdScenariosInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given scenario(s)",
Long: `Fetch and install given scenario(s) from hub`,
Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.SCENARIOS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
return cmdScenariosInstall
}
func NewCmdScenariosRemove() *cobra.Command {
var cmdScenariosRemove = &cobra.Command{
Use: "remove [config]",
Short: "Remove given scenario(s)",
Long: `remove given scenario(s)`,
Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one scenario to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
}
return nil
},
}
cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
return cmdScenariosRemove
}
func NewCmdScenariosUpgrade() *cobra.Command {
var cmdScenariosUpgrade = &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given scenario(s)",
Long: `Fetch and Upgrade given scenario(s) from hub`,
Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
}
}
return nil
},
}
cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdScenariosUpgrade
}
func NewCmdScenariosInspect() *cobra.Command {
var cmdScenariosInspect = &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given scenario",
Long: `Inspect given scenario`,
Example: `cscli scenarios inspect crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.SCENARIOS)
},
}
cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdScenariosInspect
}
func NewCmdScenariosList() *cobra.Command {
var cmdScenariosList = &cobra.Command{
Use: "list [config]",
Short: "List all scenario(s) or given one",
Long: `List all scenario(s) or given one`,
Example: `cscli scenarios list
cscli scenarios list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
},
}
cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdScenariosList
}

11
cmd/crowdsec-cli/setup.go
View file

@ -6,13 +6,15 @@ import (
"os"
"os/exec"
goccyyaml "github.com/goccy/go-yaml"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
goccyyaml "github.com/goccy/go-yaml"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
// NewSetupCmd defines the "cscli setup" command.
@ -303,7 +305,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return fmt.Errorf("while reading file %s: %w", fromFile, err)
}
if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
return err
}

12
cmd/crowdsec-cli/simulation.go
View file

@ -3,11 +3,11 @@ package main
import (
"fmt"
"os"
"slices"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -112,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
if err := csConfig.LoadSimulation(); err != nil {
log.Fatal(err)
}
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before using simulation")
}
if csConfig.Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured")
}
@ -145,18 +142,19 @@ func NewSimulationEnableCmd() *cobra.Command {
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
}
if len(args) > 0 {
for _, scenario := range args {
var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
var item = hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
continue
}
if !item.Installed {
if !item.State.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)

44
cmd/crowdsec-cli/support.go
View file

@ -58,10 +58,6 @@ func stripAnsiString(str string) string {
func collectMetrics() ([]byte, []byte, error) {
log.Info("Collecting prometheus metrics")
err := csConfig.LoadPrometheus()
if err != nil {
return nil, nil, err
}
if csConfig.Cscli.PrometheusUrl == "" {
log.Warn("No Prometheus URL configured, metrics will not be collected")
@ -69,13 +65,13 @@ func collectMetrics() ([]byte, []byte, error) {
}
humanMetrics := bytes.NewBuffer(nil)
err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
if err != nil {
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
}
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
}
@ -132,10 +128,21 @@ func collectOSInfo() ([]byte, error) {
return w.Bytes(), nil
}
func collectHubItems(itemType string) []byte {
func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
var err error
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
ListItems(out, []string{itemType}, []string{}, false, true, all)
items := make(map[string][]*cwhub.Item)
if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
if err := listItems(out, []string{itemType}, items); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes()
}
@ -157,7 +164,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
return out.Bytes(), nil
}
func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?")
}
@ -167,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
}
@ -295,7 +302,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
skipAgent = true
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
@ -333,10 +341,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
if !skipHub {
infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
}
if !skipDB {
@ -358,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
csConfig.API.Server.OnlineClient.Credentials.Password,
csConfig.API.Server.OnlineClient.Credentials.URL,
CAPIURLPrefix)
CAPIURLPrefix,
hub)
}
if !skipLAPI {
@ -366,7 +375,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
csConfig.API.Client.Credentials.Password,
csConfig.API.Client.Credentials.URL,
LAPIURLPrefix)
LAPIURLPrefix,
hub)
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
}

437
cmd/crowdsec-cli/utils.go
View file

@ -1,36 +1,17 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"slices"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/agext/levenshtein"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const MaxDistance = 7
func printHelp(cmd *cobra.Command) {
err := cmd.Help()
if err != nil {
@ -38,197 +19,6 @@ func printHelp(cmd *cobra.Command) {
}
}
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
} else {
errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
}
if ignoreErr {
log.Error(errMsg)
} else {
log.Fatalf(errMsg)
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
for _, s := range allItems {
d := levenshtein.Distance(itemName, s, nil)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := cwhub.GetInstalledItemsAsString(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
for _, itemType := range itemTypes {
itemName := ""
if len(args) == 1 {
itemName = args[0]
}
hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
}
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
if err != nil {
log.Fatalf("failed to unmarshal")
}
out.Write(x)
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
if showHeader {
header := []string{"name", "status", "version", "description"}
if showType {
header = append(header, "type")
}
err := csvwriter.Write(header)
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
for _, status := range statuses {
if status.LocalVersion == "" {
status.LocalVersion = "n/a"
}
row := []string{
status.Name,
status.Status,
status.LocalVersion,
status.Description,
}
if showType {
row = append(row, itemType)
}
err := csvwriter.Write(row)
if err != nil {
log.Fatalf("failed to write raw output : %s", err)
}
}
}
csvwriter.Flush()
}
}
func InspectItem(name string, objecitemType string) {
hubItem := cwhub.GetItem(objecitemType, name)
if hubItem == nil {
log.Fatalf("unable to retrieve item.")
}
var b []byte
var err error
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
}
fmt.Printf("%s", string(b))
if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return
}
if prometheusURL == "" {
//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
listenAddr := csConfig.Prometheus.ListenAddr
if listenAddr == "" {
listenAddr = "127.0.0.1"
}
listenPort := csConfig.Prometheus.ListenPort
if listenPort == 0 {
listenPort = 6060
}
prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
log.Debugf("No prometheus URL provided using: %s", prometheusURL)
}
fmt.Printf("\nCurrent metrics : \n")
ShowMetrics(hubItem)
}
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
/*if a range is provided, change the scope*/
@ -259,232 +49,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return nil
}
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(prometheusURL, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, item := range hubItem.Parsers {
metrics := GetParserMetric(prometheusURL, item)
parserMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Scenarios {
metrics := GetScenarioMetric(prometheusURL, item)
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
}
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}
func getDBClient() (*database.Client, error) {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
@ -518,5 +82,4 @@ func removeFromSlice(val string, slice []string) []string {
}
return slice
}

32
cmd/crowdsec-cli/utils_table.go
View file

@ -3,6 +3,7 @@ package main
import (
"fmt"
"io"
"strconv"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
@ -10,14 +11,15 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
t := newLightTable(out)
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, status := range statuses {
t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
for _, item := range items {
status, emo := item.InstallStatus()
t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.State.LocalVersion, item.State.LocalPath)
}
renderTableTitle(out, title)
t.Render()
@ -31,11 +33,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.AddRow(
fmt.Sprintf("%d", metrics["curr_count"]),
fmt.Sprintf("%d", metrics["overflow"]),
fmt.Sprintf("%d", metrics["instantiation"]),
fmt.Sprintf("%d", metrics["pour"]),
fmt.Sprintf("%d", metrics["underflow"]),
strconv.Itoa(metrics["curr_count"]),
strconv.Itoa(metrics["overflow"]),
strconv.Itoa(metrics["instantiation"]),
strconv.Itoa(metrics["pour"]),
strconv.Itoa(metrics["underflow"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@ -43,23 +45,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
}
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
skip := true
t := newTable(out)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
// don't show table if no hits
showTable := false
for source, stats := range metrics {
if stats["hits"] > 0 {
t.AddRow(
source,
fmt.Sprintf("%d", stats["hits"]),
fmt.Sprintf("%d", stats["parsed"]),
fmt.Sprintf("%d", stats["unparsed"]),
strconv.Itoa(stats["hits"]),
strconv.Itoa(stats["parsed"]),
strconv.Itoa(stats["unparsed"]),
)
skip = false
showTable = true
}
}
if !skip {
if showTable {
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
t.Render()
}

19
cmd/crowdsec/crowdsec.go
View file

@ -20,21 +20,16 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error
// Populate cwhub package tools
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, fmt.Errorf("while loading parsers: %w", err)
}
if err := LoadBuckets(cConfig); err != nil {
if err := LoadBuckets(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading scenarios: %w", err)
}
@ -44,7 +39,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
return csParsers, nil
}
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
inputEventChan = make(chan types.Event)
inputLineChan = make(chan types.Event)
@ -99,7 +94,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
log.Fatalf("starting outputs error : %s", err)
return err
}
@ -131,7 +126,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
return nil
}
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
crowdsecTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveCrowdsec")
go func() {
@ -139,7 +134,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
// this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true
if err := runCrowdsec(cConfig, parsers); err != nil {
if err := runCrowdsec(cConfig, parsers, hub); err != nil {
log.Fatalf("unable to start crowdsec routines: %s", err)
}
}()

21
cmd/crowdsec/main.go
View file

@ -75,20 +75,20 @@ type Flags struct {
type labelsMap map[string]string
func LoadBuckets(cConfig *csconfig.Config) error {
func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
var (
err error
files []string
)
for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.Installed {
files = append(files, hubScenarioItem.LocalPath)
for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.State.Installed {
files = append(files, hubScenarioItem.State.LocalPath)
}
}
buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files))
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
if err != nil {
return fmt.Errorf("scenario loading failed: %v", err)
@ -212,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
if err != nil {
return nil, err
}
if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
return nil, fmt.Errorf("unable to load configuration: common section is empty")
return nil, fmt.Errorf("while loading configuration file: %w", err)
}
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@ -228,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
dumpStates = true
}
// Configuration paths are dependency to load crowdsec configuration
if err := cConfig.LoadConfigurationPaths(); err != nil {
return nil, err
}
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
// if we're in time-machine mode, we don't want to log to file
cConfig.Common.LogMedia = "stdout"

8
cmd/crowdsec/metrics.go
View file

@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
if !config.Enabled {
return
}
if config.ListenAddr == "" {
log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
config.ListenAddr = "127.0.0.1"
}
if config.ListenPort == 0 {
log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
config.ListenPort = 6060
}
// Registering prometheus
// If in aggregated mode, do not register events associated with a source, to keep the cardinality low

7
cmd/crowdsec/output.go
View file

@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
var bucketOverflows []types.Event
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
var err error
ticker := time.NewTicker(1 * time.Second)
@ -70,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
@ -93,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)},
})
if err != nil {
return fmt.Errorf("new client api: %w", err)

19
cmd/crowdsec/serve.go
View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
}
@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
agentReady := make(chan bool, 1)
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
log.Printf("Reload is finished")
@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return fmt.Errorf("crowdsec init: %w", err)
}
// if it's just linting, we're done
if !flags.TestMode {
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
} else {
agentReady <- true

1
config/config.yaml
View file

@ -6,7 +6,6 @@ common:
log_max_size: 20
compress_logs: true
log_max_files: 10
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/

1
config/config_win.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

1
config/config_win_no_lapi.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

1
config/dev.yaml
View file

@ -2,7 +2,6 @@ common:
daemonize: true
log_media: stdout
log_level: info
working_dir: .
config_paths:
config_dir: ./config
data_dir: ./data/

1
config/user.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout
log_level: info
log_dir: /var/log/
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data

1
docker/config.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout
log_level: info
log_dir: /var/log/
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/

22
docker/docker_start.sh
View file

@ -101,19 +101,23 @@ register_bouncer() {
# $2 can be install, remove, upgrade
# $3 is a list of object names separated by space
cscli_if_clean() {
local itemtype="$1"
local action="$2"
local objs=$3
shift 3
# loop over all objects
for obj in $3; do
if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
echo "Object $1/$obj is tainted, skipping"
for obj in $objs; do
if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
echo "Object $itemtype/$obj is tainted, skipping"
else
# # Too verbose? Only show errors if not in debug mode
# if [ "$DEBUG" != "true" ]; then
# error_only=--error
# fi
error_only=""
echo "Running: cscli $error_only $1 $2 \"$obj\""
echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
# shellcheck disable=SC2086
cscli $error_only "$1" "$2" "$obj"
cscli $error_only "$itemtype" "$action" "$obj" "$@"
fi
done
}
@ -327,22 +331,22 @@ fi
## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force
fi
if [ "$DISABLE_PARSERS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean parsers remove "$DISABLE_PARSERS"
cscli_if_clean parsers remove "$DISABLE_PARSERS" --force
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
fi
## Register bouncers via env

8
docker/test/tests/test_hub_collections.py
View file

@ -30,8 +30,8 @@ def test_install_two_collections(crowdsec, flavor):
cs.wait_for_log([
# f'*collections install "{it1}"*'
# f'*collections install "{it2}"*'
f'*Enabled collections : {it1}*',
f'*Enabled collections : {it2}*',
f'*Enabled collections: {it1}*',
f'*Enabled collections: {it2}*',
])
@ -72,7 +72,7 @@ def test_install_and_disable_collection(crowdsec, flavor):
assert it not in items
logs = cs.log_lines()
# check that there was no attempt to install
assert not any(f'Enabled collections : {it}' in line for line in logs)
assert not any(f'Enabled collections: {it}' in line for line in logs)
# already done in bats, prividing here as example of a somewhat complex test
@ -91,7 +91,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor):
# implicit check for tainted=False
assert items[coll]['status'] == 'enabled'
cs.wait_for_log([
f'*Enabled collections : {coll}*',
f'*Enabled collections: {coll}*',
])
scenario = 'crowdsecurity/http-crawl-non_statics'

4
docker/test/tests/test_hub_scenarios.py
View file

@ -21,8 +21,8 @@ def test_install_two_scenarios(crowdsec, flavor):
}
with crowdsec(flavor=flavor, environment=env) as cs:
cs.wait_for_log([
f'*scenarios install "{it1}*"',
f'*scenarios install "{it2}*"',
f'*scenarios install "{it1}"*',
f'*scenarios install "{it2}"*',
"*Starting processing data*"
])
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)

2
go.mod
View file

@ -25,7 +25,7 @@ require (
github.com/c-robinson/iplib v1.0.3
github.com/cespare/xxhash/v2 v2.2.0
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.4
github.com/crowdsecurity/go-cs-lib v0.0.5
github.com/crowdsecurity/grokky v0.2.1
github.com/crowdsecurity/machineid v1.0.2
github.com/davecgh/go-spew v1.1.1

4
go.sum
View file

@ -140,8 +140,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/go-cs-lib v0.0.5 h1:eVLW+BRj3ZYn0xt5/xmgzfbbB8EBo32gM4+WpQQk2e8=
github.com/crowdsecurity/go-cs-lib v0.0.5/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=

4
pkg/csconfig/api.go
View file

@ -286,10 +286,6 @@ func (c *Config) LoadAPIServer() error {
log.Infof("loaded capi whitelist from %s: %d IPs, %d CIDRs", c.API.Server.CapiWhitelistsPath, len(c.API.Server.CapiWhitelists.Ips), len(c.API.Server.CapiWhitelists.Cidrs))
}
if err := c.LoadCommon(); err != nil {
return fmt.Errorf("loading common configuration: %s", err)
}
c.API.Server.LogDir = c.Common.LogDir
c.API.Server.LogMedia = c.Common.LogMedia
c.API.Server.CompressLogs = c.Common.CompressLogs

8
pkg/csconfig/api_test.go
View file

@ -3,7 +3,6 @@ package csconfig
import (
"net"
"os"
"path/filepath"
"strings"
"testing"
@ -142,9 +141,6 @@ func TestLoadAPIServer(t *testing.T) {
err := tmpLAPI.LoadProfiles()
require.NoError(t, err)
LogDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
logLevel := log.InfoLevel
config := &Config{}
fcontent, err := os.ReadFile("./testdata/config.yaml")
@ -179,7 +175,7 @@ func TestLoadAPIServer(t *testing.T) {
DbPath: "./testdata/test.db",
},
Common: &CommonCfg{
LogDir: "./testdata/",
LogDir: "./testdata",
LogMedia: "stdout",
},
DisableAPI: false,
@ -202,7 +198,7 @@ func TestLoadAPIServer(t *testing.T) {
ShareContext: ptr.Of(false),
ConsoleManagement: ptr.Of(false),
},
LogDir: LogDirFullPath,
LogDir: "./testdata",
LogMedia: "stdout",
OnlineClient: &OnlineApiClientCfg{
CredentialsFilePath: "./testdata/online-api-secrets.yaml",

11
pkg/csconfig/common.go
View file

@ -14,7 +14,7 @@ type CommonCfg struct {
LogMedia string `yaml:"log_media"`
LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file
LogLevel *log.Level `yaml:"log_level"`
WorkingDir string `yaml:"working_dir,omitempty"` ///var/run
WorkingDir string `yaml:"working_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
CompressLogs *bool `yaml:"compress_logs,omitempty"`
LogMaxSize int `yaml:"log_max_size,omitempty"`
LogMaxAge int `yaml:"log_max_age,omitempty"`
@ -22,15 +22,18 @@ type CommonCfg struct {
ForceColorLogs bool `yaml:"force_color_logs,omitempty"`
}
func (c *Config) LoadCommon() error {
func (c *Config) loadCommon() error {
var err error
if c.Common == nil {
return fmt.Errorf("no common block provided in configuration file")
c.Common = &CommonCfg{}
}
if c.Common.LogMedia == "" {
c.Common.LogMedia = "stdout"
}
var CommonCleanup = []*string{
&c.Common.LogDir,
&c.Common.WorkingDir,
}
for _, k := range CommonCleanup {
if *k == "" {

83
pkg/csconfig/common_test.go
View file

@ -1,83 +0,0 @@
package csconfig
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadCommon(t *testing.T) {
pidDirPath := "./testdata"
LogDirFullPath, err := filepath.Abs("./testdata/log/")
require.NoError(t, err)
WorkingDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
tests := []struct {
name string
input *Config
expected *CommonCfg
expectedErr string
}{
{
name: "basic valid configuration",
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./testdata/log/",
WorkingDir: "./testdata/",
},
},
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
LogDir: LogDirFullPath,
WorkingDir: WorkingDirFullPath,
},
},
{
name: "empty working dir",
input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./testdata",
LogMedia: "file",
LogDir: "./testdata/log/",
},
},
expected: &CommonCfg{
Daemonize: true,
PidDir: pidDirPath,
LogMedia: "file",
LogDir: LogDirFullPath,
},
},
{
name: "no common",
input: &Config{},
expected: nil,
expectedErr: "no common block provided in configuration file",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCommon()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return
}
assert.Equal(t, tc.expected, tc.input.Common)
})
}
}

36
pkg/csconfig/config.go
View file

@ -36,7 +36,7 @@ type Config struct {
PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"`
DisableAPI bool `yaml:"-"`
DisableAgent bool `yaml:"-"`
Hub *Hub `yaml:"-"`
Hub *LocalHubCfg `yaml:"-"`
}
func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {
@ -58,6 +58,37 @@ func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool
// this is actually the "merged" yaml
return nil, "", fmt.Errorf("%s: %w", configFile, err)
}
if cfg.Prometheus == nil {
cfg.Prometheus = &PrometheusCfg{}
}
if cfg.Prometheus.ListenAddr == "" {
cfg.Prometheus.ListenAddr = "127.0.0.1"
log.Debugf("prometheus.listen_addr is empty, defaulting to %s", cfg.Prometheus.ListenAddr)
}
if cfg.Prometheus.ListenPort == 0 {
cfg.Prometheus.ListenPort = 6060
log.Debugf("prometheus.listen_port is empty or zero, defaulting to %d", cfg.Prometheus.ListenPort)
}
if err = cfg.loadCommon(); err != nil {
return nil, "", err
}
if err = cfg.loadConfigurationPaths(); err != nil {
return nil, "", err
}
if err = cfg.loadHub(); err != nil {
return nil, "", err
}
if err = cfg.loadCSCLI(); err != nil {
return nil, "", err
}
return &cfg, configData, nil
}
@ -65,11 +96,8 @@ func NewDefaultConfig() *Config {
logLevel := log.InfoLevel
commonCfg := CommonCfg{
Daemonize: false,
PidDir: "/tmp/",
LogMedia: "stdout",
//LogDir unneeded
LogLevel: &logLevel,
WorkingDir: ".",
}
prometheus := PrometheusCfg{
Enabled: true,

2
pkg/csconfig/config_paths.go
View file

@ -15,7 +15,7 @@ type ConfigurationPaths struct {
NotificationDir string `yaml:"notification_dir,omitempty"`
}
func (c *Config) LoadConfigurationPaths() error {
func (c *Config) loadConfigurationPaths() error {
var err error
if c.ConfigPaths == nil {
return fmt.Errorf("no configuration paths provided")

4
pkg/csconfig/config_test.go
View file

@ -15,10 +15,10 @@ func TestNormalLoad(t *testing.T) {
require.NoError(t, err)
_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
}
func TestNewCrowdSecConfig(t *testing.T) {

15
pkg/csconfig/crowdsec_service.go
View file

@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct {
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
ContextToSend map[string][]string `yaml:"-"`
}
@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error {
return fmt.Errorf("load error (simulation): %w", err)
}
c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
c.Crowdsec.DataDir = c.ConfigPaths.DataDir
c.Crowdsec.HubDir = c.ConfigPaths.HubDir
c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
if c.Crowdsec.ParserRoutinesCount <= 0 {
c.Crowdsec.ParserRoutinesCount = 1
}
@ -145,15 +136,11 @@ func (c *Config) LoadCrowdsec() error {
return fmt.Errorf("loading api client: %s", err)
}
if err := c.LoadHub(); err != nil {
return fmt.Errorf("while loading hub: %w", err)
}
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
fallback := false
if c.Crowdsec.ConsoleContextPath == "" {
// fallback to default config file
c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
fallback = true
}

26
pkg/csconfig/crowdsec_service_test.go
View file

@ -20,18 +20,6 @@ func TestLoadCrowdsec(t *testing.T) {
acquisDirFullPath, err := filepath.Abs("./testdata/acquis")
require.NoError(t, err)
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
contextFileFullPath, err := filepath.Abs("./testdata/context.yaml")
require.NoError(t, err)
@ -66,10 +54,6 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: "",
ConsoleContextPath: contextFileFullPath,
AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
@ -109,10 +93,6 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath,
ConsoleContextPath: contextFileFullPath,
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
@ -141,7 +121,7 @@ func TestLoadCrowdsec(t *testing.T) {
},
},
Crowdsec: &CrowdsecServiceCfg{
ConsoleContextPath: contextFileFullPath,
ConsoleContextPath: "./testdata/context.yaml",
ConsoleContextValueLength: 10,
},
},
@ -149,10 +129,6 @@ func TestLoadCrowdsec(t *testing.T) {
Enable: ptr.Of(true),
AcquisitionDirPath: "",
AcquisitionFilePath: "",
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
ConsoleContextPath: contextFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,

20
pkg/csconfig/cscli.go
View file

@ -1,5 +1,9 @@
package csconfig
import (
"fmt"
)
/*cscli specific config, such as hub directory*/
type CscliCfg struct {
Output string `yaml:"output,omitempty"`
@ -7,25 +11,19 @@ type CscliCfg struct {
HubBranch string `yaml:"hub_branch"`
SimulationConfig *SimulationConfig `yaml:"-"`
DbConfig *DatabaseCfg `yaml:"-"`
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
PrometheusUrl string `yaml:"prometheus_uri"`
}
func (c *Config) LoadCSCLI() error {
func (c *Config) loadCSCLI() error {
if c.Cscli == nil {
c.Cscli = &CscliCfg{}
}
if err := c.LoadConfigurationPaths(); err != nil {
return err
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
}
c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
c.Cscli.DataDir = c.ConfigPaths.DataDir
c.Cscli.HubDir = c.ConfigPaths.HubDir
c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
return nil
}

33
pkg/csconfig/cscli_test.go
View file

@ -1,28 +1,14 @@
package csconfig
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadCSCLI(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
tests := []struct {
name string
input *Config
@ -38,26 +24,23 @@ func TestLoadCSCLI(t *testing.T) {
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
Prometheus: &PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
},
},
expected: &CscliCfg{
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
PrometheusUrl: "http://127.0.0.1:6060/metrics",
},
},
{
name: "no configuration path",
input: &Config{},
expected: &CscliCfg{},
expectedErr: "no configuration paths provided",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadCSCLI()
err := tc.input.loadCSCLI()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return

20
pkg/csconfig/hub.go
View file

@ -1,19 +1,15 @@
package csconfig
/*cscli specific config, such as hub directory*/
type Hub struct {
HubIndexFile string
HubDir string
InstallDir string
InstallDataDir string
// LocalHubCfg holds the configuration for a local hub: where to download etc.
type LocalHubCfg struct {
HubIndexFile string // Path to the local index file
HubDir string // Where the hub items are downloaded
InstallDir string // Where to install items
InstallDataDir string // Where to install data
}
func (c *Config) LoadHub() error {
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
c.Hub = &Hub{
func (c *Config) loadHub() error {
c.Hub = &LocalHubCfg{
HubIndexFile: c.ConfigPaths.HubIndexFile,
HubDir: c.ConfigPaths.HubDir,
InstallDir: c.ConfigPaths.ConfigDir,

40
pkg/csconfig/hub_test.go
View file

@ -1,32 +1,18 @@
package csconfig
import (
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadHub(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
require.NoError(t, err)
dataFullPath, err := filepath.Abs("./data")
require.NoError(t, err)
configDirFullPath, err := filepath.Abs("./testdata")
require.NoError(t, err)
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
require.NoError(t, err)
tests := []struct {
name string
input *Config
expected *Hub
expected *LocalHubCfg
expectedErr string
}{
{
@ -39,35 +25,19 @@ func TestLoadHub(t *testing.T) {
HubIndexFile: "./hub/.index.json",
},
},
expected: &Hub{
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
InstallDir: configDirFullPath,
InstallDataDir: dataFullPath,
},
},
{
name: "no data dir",
input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./testdata",
expected: &LocalHubCfg{
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
InstallDir: "./testdata",
InstallDataDir: "./data",
},
},
expectedErr: "please provide a data directory with the 'data_dir' directive in the 'config_paths' section",
},
{
name: "no configuration path",
input: &Config{},
expectedErr: "no configuration paths provided",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadHub()
err := tc.input.loadHub()
cstest.RequireErrorContains(t, err, tc.expectedErr)
if tc.expectedErr != "" {
return

11
pkg/csconfig/prometheus.go
View file

@ -1,19 +1,8 @@
package csconfig
import "fmt"
type PrometheusCfg struct {
Enabled bool `yaml:"enabled"`
Level string `yaml:"level"` //aggregated|full
ListenAddr string `yaml:"listen_addr"`
ListenPort int `yaml:"listen_port"`
}
func (c *Config) LoadPrometheus() error {
if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil {
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
}
}
return nil
}

42
pkg/csconfig/prometheus_test.go
View file

@ -1,42 +0,0 @@
package csconfig
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestLoadPrometheus(t *testing.T) {
tests := []struct {
name string
input *Config
expectedURL string
expectedErr string
}{
{
name: "basic valid configuration",
input: &Config{
Prometheus: &PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
},
Cscli: &CscliCfg{},
},
expectedURL: "http://127.0.0.1:6060",
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
err := tc.input.LoadPrometheus()
cstest.RequireErrorContains(t, err, tc.expectedErr)
require.Equal(t, tc.expectedURL, tc.input.Cscli.PrometheusUrl)
})
}
}

5
pkg/csconfig/simulation.go
View file

@ -30,11 +30,6 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
}
func (c *Config) LoadSimulation() error {
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
simCfg := SimulationConfig{}
if c.ConfigPaths.SimulationFilePath == "" {
c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")

13
pkg/csconfig/simulation_test.go
View file

@ -2,7 +2,6 @@ package csconfig
import (
"fmt"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
@ -12,12 +11,6 @@ import (
)
func TestSimulationLoading(t *testing.T) {
testXXFullPath, err := filepath.Abs("./testdata/xxx.yaml")
require.NoError(t, err)
badYamlFullPath, err := filepath.Abs("./testdata/config.yaml")
require.NoError(t, err)
tests := []struct {
name string
input *Config
@ -56,7 +49,7 @@ func TestSimulationLoading(t *testing.T) {
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedErr: fmt.Sprintf("while reading yaml file: open %s: %s", testXXFullPath, cstest.FileNotFoundMessage),
expectedErr: fmt.Sprintf("while reading yaml file: open ./testdata/xxx.yaml: %s", cstest.FileNotFoundMessage),
},
{
name: "basic bad file content",
@ -67,7 +60,7 @@ func TestSimulationLoading(t *testing.T) {
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
},
{
name: "basic bad file content",
@ -78,7 +71,7 @@ func TestSimulationLoading(t *testing.T) {
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedErr: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
expectedErr: "while unmarshaling simulation file './testdata/config.yaml' : yaml: unmarshal errors",
},
}

1
pkg/csconfig/testdata/config.yaml vendored
View file

@ -2,7 +2,6 @@ common:
daemonize: false
log_media: stdout
log_level: info
working_dir: .
prometheus:
enabled: true
level: full

284
pkg/cwhub/cwhub.go
View file

@ -2,287 +2,31 @@ package cwhub
import (
"fmt"
"os"
"net/http"
"path/filepath"
"sort"
"strings"
"github.com/enescakir/emoji"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"time"
)
const (
HubIndexFile = ".index.json"
// managed item types
PARSERS = "parsers"
PARSERS_OVFLW = "postoverflows"
SCENARIOS = "scenarios"
COLLECTIONS = "collections"
)
var (
ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS}
ErrMissingReference = errors.New("Reference(s) missing in collection")
// XXX: can we remove these globals?
skippedLocal = 0
skippedTainted = 0
RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
HubBranch = "master"
hubIdx map[string]map[string]Item
)
type ItemVersion struct {
Digest string `json:"digest,omitempty"` // meow
Deprecated bool `json:"deprecated,omitempty"`
var hubClient = &http.Client{
Timeout: 120 * time.Second,
}
type ItemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
// Item can be: parser, scenario, collection..
type Item struct {
// descriptive info
Type string `json:"type,omitempty" yaml:"type,omitempty"` // parser|postoverflows|scenario|collection(|enrich)
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
Name string `json:"name,omitempty"` // as seen in .config.json, usually "author/name"
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .config.json
Author string `json:"author,omitempty"` // as seen in .config.json
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .config.json
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
// remote (hub) info
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // the path relative to (git | hub API) ie. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty"` // the last version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // the list of existing versions
// local (deployed) info
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` // the local meow
Installed bool `json:"installed,omitempty"`
Downloaded bool `json:"downloaded,omitempty"`
UpToDate bool `json:"up_to_date,omitempty"`
Tainted bool `json:"tainted,omitempty"` // has it been locally modified
Local bool `json:"local,omitempty"` // if it's a non versioned control one
// if it's a collection, it's not a single file
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
func (i *Item) status() (string, emoji.Emoji) {
status := "disabled"
ok := false
if i.Installed {
ok = true
status = "enabled"
}
managed := true
if i.Local {
managed = false
status += ",local"
}
warning := false
if i.Tainted {
warning = true
status += ",tainted"
} else if !i.UpToDate && !i.Local {
warning = true
status += ",update-available"
}
emo := emoji.QuestionMark
switch {
case !managed:
emo = emoji.House
case !i.Installed:
emo = emoji.Prohibited
case warning:
emo = emoji.Warning
case ok:
emo = emoji.CheckMark
}
return status, emo
}
func (i *Item) hubStatus() ItemHubStatus {
status, emo := i.status()
return ItemHubStatus{
Name: i.Name,
LocalVersion: i.LocalVersion,
LocalPath: i.LocalPath,
Description: i.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", emo, status),
}
}
// versionStatus: semver requires 'v' prefix
func (i *Item) versionStatus() int {
return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
}
func GetItemMap(itemType string) map[string]Item {
m, ok := hubIdx[itemType]
if !ok {
return nil
}
return m
}
// Given a FileInfo, extract the map key. Follow a symlink if necessary
func itemKey(itemPath string) (string, error) {
f, err := os.Lstat(itemPath)
// safePath returns a joined path and ensures that it does not escape the base directory.
func safePath(dir, filePath string) (string, error) {
absBaseDir, err := filepath.Abs(filepath.Clean(dir))
if err != nil {
return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
return "", err
}
if f.Mode()&os.ModeSymlink == 0 {
// it's not a symlink, so the filename itsef should be the key
return filepath.Base(itemPath), nil
}
// resolve the symlink to hub file
pathInHub, err := os.Readlink(itemPath)
absFilePath, err := filepath.Abs(filepath.Join(dir, filePath))
if err != nil {
return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
return "", err
}
author := filepath.Base(filepath.Dir(pathInHub))
if !strings.HasPrefix(absFilePath, absBaseDir) {
return "", fmt.Errorf("path %s escapes base directory %s", filePath, dir)
}
fname := filepath.Base(pathInHub)
fname = strings.TrimSuffix(fname, ".yaml")
fname = strings.TrimSuffix(fname, ".yml")
return fmt.Sprintf("%s/%s", author, fname), nil
}
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
itemKey, err := itemKey(itemPath)
if err != nil {
return nil, err
}
m := GetItemMap(itemType)
if m == nil {
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
}
v, ok := m[itemKey]
if !ok {
return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
}
return &v, nil
}
func GetItem(itemType string, itemName string) *Item {
if m, ok := GetItemMap(itemType)[itemName]; ok {
return &m
}
return nil
}
func AddItem(itemType string, item Item) error {
for _, itype := range ItemTypes {
if itype == itemType {
hubIdx[itemType][item.Name] = item
return nil
}
}
return fmt.Errorf("ItemType %s is unknown", itemType)
}
func DisplaySummary() {
log.Infof("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
if skippedLocal > 0 || skippedTainted > 0 {
log.Infof("unmanaged items: %d local, %d tainted", skippedLocal, skippedTainted)
}
}
func GetInstalledItems(itemType string) ([]Item, error) {
items, ok := hubIdx[itemType]
if !ok {
return nil, fmt.Errorf("no %s in hubIdx", itemType)
}
retItems := make([]Item, 0)
for _, item := range items {
if item.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
func GetInstalledItemsAsString(itemType string) ([]string, error) {
items, err := GetInstalledItems(itemType)
if err != nil {
return nil, err
}
retStr := make([]string, len(items))
for i, it := range items {
retStr[i] = it.Name
}
return retStr, nil
}
// Returns a slice of entries for packages: name, status, local_path, local_version, utf8_status (fancy)
func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
if _, ok := hubIdx[itemType]; !ok {
log.Errorf("type %s doesn't exist", itemType)
return nil
}
ret := make([]ItemHubStatus, 0)
// remember, you do it for the user :)
for _, item := range hubIdx[itemType] {
if name != "" && name != item.Name {
// user has requested a specific name
continue
}
// Only enabled items ?
if !all && !item.Installed {
continue
}
// Check the item status
ret = append(ret, item.hubStatus())
}
sort.Slice(ret, func(i, j int) bool { return ret[i].Name < ret[j].Name })
return ret
return absFilePath, nil
}

326
pkg/cwhub/cwhub_test.go
View file

@ -9,14 +9,13 @@ import (
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
/*
To test :
- Download 'first' hub index
@ -28,294 +27,63 @@ import (
var responseByPath map[string]string
func TestItemStatus(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub)
require.NoError(t, err, "failed to load hub index")
// get existing map
x := GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item : good and bad
for k := range x {
item := GetItem(COLLECTIONS, k)
require.NotNil(t, item)
item.Installed = true
item.UpToDate = false
item.Local = false
item.Tainted = false
txt, _ := item.status()
require.Equal(t, "enabled,update-available", txt)
item.Installed = false
item.UpToDate = false
item.Local = true
item.Tainted = false
txt, _ = item.status()
require.Equal(t, "disabled,local", txt)
}
DisplaySummary()
}
func TestGetters(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub)
require.NoError(t, err, "failed to load hub index")
// get non existing map
empty := GetItemMap("ratata")
require.Nil(t, empty)
// get existing map
x := GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item : good and bad
for k := range x {
empty := GetItem(COLLECTIONS, k+"nope")
require.Nil(t, empty)
item := GetItem(COLLECTIONS, k)
require.NotNil(t, item)
// Add item and get it
item.Name += "nope"
err := AddItem(COLLECTIONS, *item)
// testHub initializes a temporary hub with an empty json file, optionally updating it.
func testHub(t *testing.T, update bool) *Hub {
tmpDir, err := os.MkdirTemp("", "testhub")
require.NoError(t, err)
newitem := GetItem(COLLECTIONS, item.Name)
require.NotNil(t, newitem)
err = AddItem("ratata", *item)
cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
local := &csconfig.LocalHubCfg{
HubDir: filepath.Join(tmpDir, "crowdsec", "hub"),
HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"),
InstallDir: filepath.Join(tmpDir, "crowdsec"),
InstallDataDir: filepath.Join(tmpDir, "installed-data"),
}
}
func TestIndexDownload(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
err = os.MkdirAll(local.HubDir, 0o700)
require.NoError(t, err)
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
require.NoError(t, err, "failed to download index")
err = os.MkdirAll(local.InstallDir, 0o700)
require.NoError(t, err)
err = GetHubIdx(cfg.Hub)
require.NoError(t, err, "failed to load hub index")
}
err = os.MkdirAll(local.InstallDataDir, 0o700)
require.NoError(t, err)
func getTestCfg() *csconfig.Config {
cfg := &csconfig.Config{Hub: &csconfig.Hub{}}
cfg.Hub.InstallDir, _ = filepath.Abs("./install")
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
return cfg
}
func envSetup(t *testing.T) *csconfig.Config {
resetResponseByPath()
log.SetLevel(log.DebugLevel)
cfg := getTestCfg()
defaultTransport := http.DefaultClient.Transport
err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644)
require.NoError(t, err)
t.Cleanup(func() {
http.DefaultClient.Transport = defaultTransport
os.RemoveAll(tmpDir)
})
remote := &RemoteHubCfg{
Branch: "master",
URLTemplate: mockURLTemplate,
IndexPath: ".index.json",
}
hub, err := NewHub(local, remote, update)
require.NoError(t, err)
return hub
}
// envSetup initializes the temporary hub and mocks the http client.
func envSetup(t *testing.T) *Hub {
setResponseByPath()
log.SetLevel(log.DebugLevel)
defaultTransport := hubClient.Transport
t.Cleanup(func() {
hubClient.Transport = defaultTransport
})
// Mock the http client
http.DefaultClient.Transport = newMockTransport()
hubClient.Transport = newMockTransport()
err := os.MkdirAll(cfg.Hub.InstallDir, 0700)
require.NoError(t, err)
hub := testHub(t, true)
err = os.MkdirAll(cfg.Hub.HubDir, 0700)
require.NoError(t, err)
err = UpdateHubIdx(cfg.Hub)
require.NoError(t, err)
// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
// log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
// }
// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
// log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
// }
return cfg
}
func envTearDown(cfg *csconfig.Config) {
if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
}
if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
}
}
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
// Install the parser
err := DownloadLatest(cfg, &item, false, false)
require.NoError(t, err, "failed to download %s", item.Name)
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
err = EnableItem(cfg, &item)
require.NoError(t, err, "failed to enable %s", item.Name)
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
}
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
defer f.Close()
_, err = f.WriteString("tainted")
require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
// Local sync and check status
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
}
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.False(t, hubIdx[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
err := DownloadLatest(cfg, &item, true, true)
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status
_, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
assert.True(t, hubIdx[item.Type][item.Name].Installed, "%s should be installed", item.Name)
// Remove
err := DisableItem(cfg, &item, false, false)
require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status
warns, err := LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings : %+v", warns)
assert.False(t, hubIdx[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hubIdx[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
// Purge
err = DisableItem(cfg, &item, true, false)
require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status
warns, err = LocalSync(cfg)
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings : %+v", warns)
assert.False(t, hubIdx[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hubIdx[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
}
func TestInstallParser(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
cfg := envSetup(t)
defer envTearDown(cfg)
getHubIdxOrFail(t)
// map iteration is random by itself
for _, it := range hubIdx[PARSERS] {
testInstallItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
testTaintItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = GetHubStatusForItemType(PARSERS, it.Name, false)
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
testDisableItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
break
}
}
func TestInstallCollection(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
cfg := envSetup(t)
defer envTearDown(cfg)
getHubIdxOrFail(t)
// map iteration is random by itself
for _, it := range hubIdx[COLLECTIONS] {
testInstallItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testTaintItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testDisableItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
x := GetHubStatusForItemType(COLLECTIONS, it.Name, false)
log.Infof("%+v", x)
break
}
return hub
}
type mockTransport struct{}
@ -324,7 +92,7 @@ func newMockTransport() http.RoundTripper {
return &mockTransport{}
}
// Implement http.RoundTripper
// Implement http.RoundTripper.
func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// Create mocked http.Response
response := &http.Response{
@ -362,7 +130,7 @@ func fileToStringX(path string) string {
return strings.ReplaceAll(string(data), "\r\n", "\n")
}
func resetResponseByPath() {
func setResponseByPath() {
responseByPath = map[string]string{
"/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),
"/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml": fileToStringX("./testdata/foobar_parser.yaml"),

72
pkg/cwhub/dataset.go
View file

@ -1,70 +1,84 @@
package cwhub
import (
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// The DataSet is a list of data sources required by an item (built from the data: section in the yaml).
type DataSet struct {
Data []*types.DataSource `yaml:"data,omitempty"`
Data []types.DataSource `yaml:"data,omitempty"`
}
// downloadFile downloads a file and writes it to disk, with no hash verification.
func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath)
req, err := http.NewRequest(http.MethodGet, url, nil)
resp, err := hubClient.Get(url)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
return fmt.Errorf("while downloading %s: %w", url, err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body))
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
file, err := os.Create(destPath)
if err != nil {
return err
}
defer file.Close()
// avoid reading the whole file in memory
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
}
_, err = file.Write(body)
if err != nil {
return err
}
err = file.Sync()
if err != nil {
if err = file.Sync(); err != nil {
return err
}
return nil
}
func GetData(data []*types.DataSource, dataDir string) error {
for _, dataS := range data {
destPath := filepath.Join(dataDir, dataS.DestPath)
// downloadDataSet downloads all the data files for an item.
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
dec := yaml.NewDecoder(reader)
for {
data := &DataSet{}
if err := dec.Decode(data); err != nil {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("while reading file: %w", err)
}
for _, dataS := range data.Data {
destPath, err := safePath(dataFolder, dataS.DestPath)
if err != nil {
return err
}
if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
err := downloadFile(dataS.SourceURL, destPath)
if err != nil {
return err
if err := downloadFile(dataS.SourceURL, destPath); err != nil {
return fmt.Errorf("while getting data: %w", err)
}
}
}
}

17
pkg/cwhub/dataset_test.go
View file

@ -6,6 +6,7 @@ import (
"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDownloadFile(t *testing.T) {
@ -14,12 +15,14 @@ func TestDownloadFile(t *testing.T) {
httpmock.Activate()
defer httpmock.DeactivateAndReset()
//OK
httpmock.RegisterResponder(
"GET",
"https://example.com/xx",
httpmock.NewStringResponder(200, "example content oneoneone"),
)
httpmock.RegisterResponder(
"GET",
"https://example.com/x",
@ -27,17 +30,21 @@ func TestDownloadFile(t *testing.T) {
)
err := downloadFile("https://example.com/xx", examplePath)
assert.NoError(t, err)
require.NoError(t, err)
content, err := os.ReadFile(examplePath)
assert.Equal(t, "example content oneoneone", string(content))
assert.NoError(t, err)
require.NoError(t, err)
//bad uri
err = downloadFile("https://zz.com", examplePath)
assert.Error(t, err)
require.Error(t, err)
//404
err = downloadFile("https://example.com/x", examplePath)
assert.Error(t, err)
require.Error(t, err)
//bad target
err = downloadFile("https://example.com/xx", "")
assert.Error(t, err)
require.Error(t, err)
}

113
pkg/cwhub/doc.go Normal file
View file

@ -0,0 +1,113 @@
// Package cwhub is responsible for installing and upgrading the local hub files for CrowdSec.
//
// # Definitions
//
// - A hub ITEM is a file that defines a parser, a scenario, a collection... in the case of a collection, it has dependencies on other hub items.
// - The hub INDEX is a JSON file that contains a tree of available hub items.
// - A REMOTE HUB is an HTTP server that hosts the hub index and the hub items. It can serve from several branches, usually linked to the CrowdSec version.
// - A LOCAL HUB is a directory that contains a copy of the hub index and the downloaded hub items.
//
// Once downloaded, hub items can be installed by linking to them from the configuration directory.
// If an item is present in the configuration directory but it's not a link to the local hub, it is
// considered as a LOCAL ITEM and won't be removed or upgraded.
//
// # Directory Structure
//
// A typical directory layout is the following:
//
// For the local hub (HubDir = /etc/crowdsec/hub):
//
// - /etc/crowdsec/hub/.index.json
// - /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
// - /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
//
// For the configuration directory (InstallDir = /etc/crowdsec):
//
// - /etc/crowdsec/parsers/{stage}/{parser-name.yaml} -> /etc/crowdsec/hub/parsers/{stage}/{author}/{parser-name}.yaml
// - /etc/crowdsec/scenarios/{scenario-name.yaml} -> /etc/crowdsec/hub/scenarios/{author}/{scenario-name}.yaml
// - /etc/crowdsec/scenarios/local-scenario.yaml
//
// Note that installed items are not grouped by author, this may change in the future if we want to
// support items with the same name from different authors.
//
// Only parsers and postoverflows have the concept of stage.
//
// Additionally, an item can reference a DATA SET that is installed in a different location than
// the item itself. These files are stored in the data directory (InstallDataDir = /var/lib/crowdsec/data).
//
// - /var/lib/crowdsec/data/http_path_traversal.txt
// - /var/lib/crowdsec/data/jira_cve_2021-26086.txt
// - /var/lib/crowdsec/data/log4j2_cve_2021_44228.txt
// - /var/lib/crowdsec/data/sensitive_data.txt
//
//
// # Using the package
//
// The main entry point is the Hub struct. You can create a new instance with NewHub().
// This constructor takes three parameters, but only the LOCAL HUB configuration is required:
//
// import (
// "fmt"
// "github.com/crowdsecurity/crowdsec/pkg/csconfig"
// "github.com/crowdsecurity/crowdsec/pkg/cwhub"
// )
//
// localHub := csconfig.LocalHubCfg{
// HubIndexFile: "/etc/crowdsec/hub/.index.json",
// HubDir: "/etc/crowdsec/hub",
// InstallDir: "/etc/crowdsec",
// InstallDataDir: "/var/lib/crowdsec/data",
// }
// hub, err := cwhub.NewHub(localHub, nil, false)
// if err != nil {
// return fmt.Errorf("unable to initialize hub: %w", err)
// }
//
// Now you can use the hub to access the existing items:
//
// // list all the parsers
// for _, parser := range hub.GetItemMap(cwhub.PARSERS) {
// fmt.Printf("parser: %s\n", parser.Name)
// }
//
// // retrieve a specific collection
// coll := hub.GetItem(cwhub.COLLECTIONS, "crowdsecurity/linux")
// if coll == nil {
// return fmt.Errorf("collection not found")
// }
//
// You can also install items if they have already been downloaded:
//
// // install a parser
// force := false
// downloadOnly := false
// err := parser.Install(force, downloadOnly)
// if err != nil {
// return fmt.Errorf("unable to install parser: %w", err)
// }
//
// As soon as you try to install an item that is not downloaded or is not up-to-date (meaning its computed hash
// does not correspond to the latest version available in the index), a download will be attempted and you'll
// get the error "remote hub configuration is not provided".
//
// To provide the remote hub configuration, use the second parameter of NewHub():
//
// remoteHub := cwhub.RemoteHubCfg{
// URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// Branch: "master",
// IndexPath: ".index.json",
// }
// updateIndex := false
// hub, err := cwhub.NewHub(localHub, remoteHub, updateIndex)
// if err != nil {
// return fmt.Errorf("unable to initialize hub: %w", err)
// }
//
// The URLTemplate is a string that will be used to build the URL of the remote hub. It must contain two
// placeholders: the branch and the file path (it will be an index or an item).
//
// Setting the third parameter to true will download the latest version of the index, if available on the
// specified branch.
// There is no exported method to update the index once the hub struct is created.
//
package cwhub

324
pkg/cwhub/download.go
View file

@ -1,324 +0,0 @@
package cwhub
import (
"bytes"
"crypto/sha256"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
var ErrIndexNotFound = fmt.Errorf("index not found")
func UpdateHubIdx(hub *csconfig.Hub) error {
bidx, err := DownloadHubIdx(hub)
if err != nil {
return fmt.Errorf("failed to download index: %w", err)
}
ret, err := LoadPkgIndex(bidx)
if err != nil {
if !errors.Is(err, ErrMissingReference) {
return fmt.Errorf("failed to read index: %w", err)
}
}
hubIdx = ret
if _, err := LocalSync(hub); err != nil {
return fmt.Errorf("failed to sync: %w", err)
}
return nil
}
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
if err != nil {
return nil, fmt.Errorf("failed to build request for hub index: %w", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("failed http request for hub index: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
return nil, ErrIndexNotFound
}
return nil, fmt.Errorf("bad http code %d while requesting %s", resp.StatusCode, req.URL.String())
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
}
oldContent, err := os.ReadFile(hub.HubIndexFile)
if err != nil {
if !os.IsNotExist(err) {
log.Warningf("failed to read hub index: %s", err)
}
} else if bytes.Equal(body, oldContent) {
log.Info("hub index is up to date")
// write it anyway, can't hurt
}
file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, fmt.Errorf("while opening hub index file: %w", err)
}
defer file.Close()
wsize, err := file.Write(body)
if err != nil {
return nil, fmt.Errorf("while writing hub index file: %w", err)
}
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
return body, nil
}
// DownloadLatest will download the latest version of Item to the tdir directory
func DownloadLatest(hub *csconfig.Hub, target *Item, overwrite bool, updateOnly bool) error {
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
if target.Type != COLLECTIONS {
if !target.Installed && updateOnly && target.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name)
return nil
}
return DownloadItem(hub, target, overwrite)
}
// collection
var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
for idx, ptr := range tmp {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx[ptrtype][p]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
if !val.Installed && updateOnly && val.Downloaded {
log.Debugf("skipping upgrade of %s : not installed", target.Name)
continue
}
log.Debugf("Download %s sub-item : %s %s (%t -> %t)", target.Name, ptrtype, p, target.Installed, updateOnly)
//recurse as it's a collection
if ptrtype == COLLECTIONS {
log.Tracef("collection, recurse")
err = DownloadLatest(hub, &val, overwrite, updateOnly)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
}
downloaded := val.Downloaded
err = DownloadItem(hub, &val, overwrite)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !val.Installed && !downloaded {
if err = EnableItem(hub, &val); err != nil {
return fmt.Errorf("enabling '%s': %w", val.Name, err)
}
}
hubIdx[ptrtype][p] = val
}
}
err = DownloadItem(hub, target, overwrite)
if err != nil {
return fmt.Errorf("failed to download item: %w", err)
}
return nil
}
func DownloadItem(hub *csconfig.Hub, target *Item, overwrite bool) error {
tdir := hub.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
if target.Tainted {
log.Debugf("%s : tainted, not updated", target.Name)
return nil
}
if target.UpToDate {
// We still have to check if data files are present
log.Debugf("%s : up-to-date, not updated", target.Name)
}
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
if err != nil {
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
}
h := sha256.New()
if _, err = h.Write(body); err != nil {
return fmt.Errorf("while hashing %s: %w", target.Name, err)
}
meow := fmt.Sprintf("%x", h.Sum(nil))
if meow != target.Versions[target.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
return fmt.Errorf("invalid download hash for %s", target.Name)
}
//all good, install
//check if parent dir exists
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
// ensure that target file is within target dir
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
}
if !strings.HasPrefix(finalPath, tdir) {
return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
}
// check dir
if _, err = os.Stat(parentDir); os.IsNotExist(err) {
log.Debugf("%s doesn't exist, create", parentDir)
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating parent directories: %w", err)
}
}
// check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s : overwrite", target.Name)
log.Debugf("target: %s/%s", tdir, target.RemotePath)
} else {
log.Infof("%s : OK", target.Name)
}
f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("while opening file: %w", err)
}
defer f.Close()
_, err = f.Write(body)
if err != nil {
return fmt.Errorf("while writing file: %w", err)
}
target.Downloaded = true
target.Tainted = false
target.UpToDate = true
if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
}
hubIdx[target.Type][target.Name] = *target
return nil
}
func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
itemFile, err := os.Open(itemFilePath)
if err != nil {
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
}
defer itemFile.Close()
if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}
return nil
}
func downloadData(dataFolder string, force bool, reader io.Reader) error {
var err error
dec := yaml.NewDecoder(reader)
for {
data := &DataSet{}
err = dec.Decode(data)
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("while reading file: %w", err)
}
download := false
for _, dataS := range data.Data {
if _, err = os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
download = true
}
}
if download || force {
err = GetData(data.Data, dataFolder)
if err != nil {
return fmt.Errorf("while getting data: %w", err)
}
}
}
return nil
}

52
pkg/cwhub/download_test.go
View file

@ -1,52 +0,0 @@
package cwhub
import (
"fmt"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func TestDownloadHubIdx(t *testing.T) {
back := RawFileURLTemplate
// bad url template
fmt.Println("Test 'bad URL'")
RawFileURLTemplate = "x"
ret, err := DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
log.Errorf("unexpected error %s", err)
}
fmt.Printf("->%+v", ret)
// bad domain
fmt.Println("Test 'bad domain'")
RawFileURLTemplate = "https://baddomain/%s/%s"
ret, err = DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
log.Errorf("unexpected error %s", err)
}
fmt.Printf("->%+v", ret)
// bad target path
fmt.Println("Test 'bad target path'")
RawFileURLTemplate = back
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
log.Errorf("unexpected error %s", err)
}
RawFileURLTemplate = back
fmt.Printf("->%+v", ret)
}

190
pkg/cwhub/enable.go Normal file
View file

@ -0,0 +1,190 @@
package cwhub
// Enable/disable items already downloaded
import (
"fmt"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
)
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
// (eg. /etc/crowdsec/collections/xyz.yaml).
// Raises an error if the path goes outside of the install dir.
func (i *Item) installPath() (string, error) {
p := i.Type
if i.Stage != "" {
p = filepath.Join(p, i.Stage)
}
return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
}
// downloadPath returns the location of the actual config file in the hub
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
// Raises an error if the path goes outside of the hub dir.
func (i *Item) downloadPath() (string, error) {
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
if err != nil {
return "", err
}
return ret, nil
}
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
func (i *Item) createInstallLink() error {
dest, err := i.installPath()
if err != nil {
return err
}
destDir := filepath.Dir(dest)
if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating %s: %w", destDir, err)
}
if _, err = os.Lstat(dest); !os.IsNotExist(err) {
log.Infof("%s already exists.", dest)
return nil
}
src, err := i.downloadPath()
if err != nil {
return err
}
if err = os.Symlink(src, dest); err != nil {
return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
}
return nil
}
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
func (i *Item) enable() error {
if i.State.Installed {
if i.State.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
}
if i.IsLocal() {
return fmt.Errorf("%s is local, won't enable", i.Name)
}
// if it's a collection, check sub-items even if the collection file itself is up-to-date
if i.State.UpToDate && !i.HasSubItems() {
log.Tracef("%s is installed and up-to-date, skip.", i.Name)
return nil
}
}
for _, sub := range i.SubItems() {
if err := sub.enable(); err != nil {
return fmt.Errorf("while installing %s: %w", sub.Name, err)
}
}
if err := i.createInstallLink(); err != nil {
return err
}
log.Infof("Enabled %s: %s", i.Type, i.Name)
i.State.Installed = true
return nil
}
// purge removes the actual config file that was downloaded.
func (i *Item) purge() error {
if !i.State.Downloaded {
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
return nil
}
src, err := i.downloadPath()
if err != nil {
return err
}
if err := os.Remove(src); err != nil {
if os.IsNotExist(err) {
log.Debugf("%s doesn't exist, no need to remove", src)
return nil
}
return fmt.Errorf("while removing file: %w", err)
}
i.State.Downloaded = false
log.Infof("Removed source file [%s]: %s", i.Name, src)
return nil
}
// removeInstallLink removes the symlink to the downloaded content.
func (i *Item) removeInstallLink() error {
syml, err := i.installPath()
if err != nil {
return err
}
stat, err := os.Lstat(syml)
if err != nil {
return err
}
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
hubpath, err := os.Readlink(syml)
if err != nil {
return fmt.Errorf("while reading symlink: %w", err)
}
src, err := i.downloadPath()
if err != nil {
return err
}
if hubpath != src {
log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
if err := os.Remove(syml); err != nil {
return fmt.Errorf("while removing symlink: %w", err)
}
log.Infof("Removed symlink [%s]: %s", i.Name, syml)
return nil
}
// disable removes the install link, and optionally the downloaded content.
func (i *Item) disable(purge bool, force bool) error {
err := i.removeInstallLink()
if os.IsNotExist(err) {
if !purge && !force {
link, _ := i.installPath()
return fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
}
} else if err != nil {
return err
}
i.State.Installed = false
if purge {
if err := i.purge(); err != nil {
return err
}
}
return nil
}

141
pkg/cwhub/enable_test.go Normal file
View file

@ -0,0 +1,141 @@
package cwhub
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func testInstall(hub *Hub, t *testing.T, item *Item) {
// Install the parser
_, err := item.downloadLatest(false, false)
require.NoError(t, err, "failed to download %s", item.Name)
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
err = item.enable()
require.NoError(t, err, "failed to enable %s", item.Name)
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
}
func testTaint(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
// truncate the file
f, err := os.Create(item.State.LocalPath)
require.NoError(t, err)
f.Close()
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name)
}
func testUpdate(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
_, err := item.downloadLatest(true, true)
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisable(hub *Hub, t *testing.T, item *Item) {
assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
// Remove
err := item.disable(false, false)
require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name)
// Purge
err = item.disable(true, false)
require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name)
}
func TestInstallParser(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
hub := envSetup(t)
// map iteration is random by itself
for _, it := range hub.Items[PARSERS] {
testInstall(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testTaint(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testUpdate(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testDisable(hub, t, it)
break
}
}
func TestInstallCollection(t *testing.T) {
/*
- install a random parser
- check its status
- taint it
- check its status
- force update it
- check its status
- remove it
*/
hub := envSetup(t)
// map iteration is random by itself
for _, it := range hub.Items[COLLECTIONS] {
testInstall(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testTaint(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testUpdate(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testDisable(hub, t, it)
break
}
}

21
pkg/cwhub/errors.go Normal file
View file

@ -0,0 +1,21 @@
package cwhub
import (
"errors"
"fmt"
)
var (
// ErrNilRemoteHub is returned when the remote hub configuration is not provided to the NewHub constructor.
ErrNilRemoteHub = errors.New("remote hub configuration is not provided. Please report this issue to the developers")
)
// IndexNotFoundError is returned when the remote hub index is not found.
type IndexNotFoundError struct {
URL string
Branch string
}
func (e IndexNotFoundError) Error() string {
return fmt.Sprintf("index not found at %s, branch '%s'. Please check the .cscli.hub_branch value if you specified it in config.yaml, or use 'master' if not sure", e.URL, e.Branch)
}

455
pkg/cwhub/helpers.go
View file

@ -1,222 +1,373 @@
package cwhub
// Install, upgrade and remove items from the hub to the local configuration
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"slices"
)
// pick a hub branch corresponding to the current crowdsec version.
func chooseHubBranch() string {
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
//lint:ignore nilerr
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("current version is equal to latest (%s)", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using master branch for the hub")
return "master"
}
log.Warnf("Crowdsec is not the latest version. "+
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
csVersion, latest)
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
"added to Crowdsec Hub after CrowdSec %s", latest)
return csVersion
}
// SetHubBranch sets the package variable that points to the hub branch.
func SetHubBranch() {
// a branch is already set, or specified from the flags
if HubBranch != "" {
return
}
// use the branch corresponding to the crowdsec version
HubBranch = chooseHubBranch()
log.Debugf("Using branch '%s' for the hub", HubBranch)
}
func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bool, downloadOnly bool) error {
item := GetItem(obtype, name)
if item == nil {
return fmt.Errorf("unable to retrieve item: %s", name)
}
if downloadOnly && item.Downloaded && item.UpToDate {
log.Warningf("%s is already downloaded and up-to-date", item.Name)
// Install installs the item from the hub, downloading it if needed.
func (i *Item) Install(force bool, downloadOnly bool) error {
if downloadOnly && i.State.Downloaded && i.State.UpToDate {
log.Infof("%s is already downloaded and up-to-date", i.Name)
if !force {
return nil
}
}
err := DownloadLatest(csConfig.Hub, item, force, true)
filePath, err := i.downloadLatest(force, true)
if err != nil {
return fmt.Errorf("while downloading %s: %w", item.Name, err)
}
if err = AddItem(obtype, *item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
return fmt.Errorf("while downloading %s: %w", i.Name, err)
}
if downloadOnly {
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath))
log.Infof("Downloaded %s to %s", i.Name, filePath)
return nil
}
err = EnableItem(csConfig.Hub, item)
if err != nil {
return fmt.Errorf("while enabling %s: %w", item.Name, err)
if err := i.enable(); err != nil {
return fmt.Errorf("while enabling %s: %w", i.Name, err)
}
if err := AddItem(obtype, *item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
}
log.Infof("Enabled %s", item.Name)
log.Infof("Enabled %s", i.Name)
return nil
}
// XXX this must return errors instead of log.Fatal
func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) {
if name != "" {
item := GetItem(itemType, name)
// descendants returns a list of all (direct or indirect) dependencies of the item.
func (i *Item) descendants() ([]*Item, error) {
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
if item == nil {
log.Fatalf("unable to retrieve: %s", name)
return nil
}
err := DisableItem(csConfig.Hub, item, purge, forceAction)
if visited[item] {
return nil
}
visited[item] = true
for _, subItem := range item.SubItems() {
if subItem == i {
return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
}
*result = append(*result, subItem)
err := collectSubItems(subItem, visited, result)
if err != nil {
log.Fatalf("unable to disable %s : %v", item.Name, err)
return err
}
}
if err = AddItem(itemType, *item); err != nil {
log.Fatalf("unable to add %s: %v", item.Name, err)
return nil
}
return
}
ret := []*Item{}
visited := map[*Item]bool{}
if !all {
log.Fatal("removing item: no item specified")
}
disabled := 0
// remove all
for _, v := range GetItemMap(itemType) {
if !v.Installed {
continue
}
err := DisableItem(csConfig.Hub, &v, purge, forceAction)
err := collectSubItems(i, visited, &ret)
if err != nil {
log.Fatalf("unable to disable %s : %v", v.Name, err)
return nil, err
}
if err := AddItem(itemType, v); err != nil {
log.Fatalf("unable to add %s: %v", v.Name, err)
}
disabled++
}
log.Infof("Disabled %d items", disabled)
return ret, nil
}
func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) {
updated := 0
found := false
// Remove disables the item, optionally removing the downloaded content.
func (i *Item) Remove(purge bool, force bool) (bool, error) {
if i.IsLocal() {
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
}
for _, v := range GetItemMap(itemType) {
if name != "" && name != v.Name {
if i.State.Tainted && !force {
return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
}
if !i.State.Installed && !purge {
log.Infof("removing %s: not installed -- no need to remove", i.Name)
return false, nil
}
removed := false
descendants, err := i.descendants()
if err != nil {
return false, err
}
ancestors := i.Ancestors()
for _, sub := range i.SubItems() {
if !sub.State.Installed {
continue
}
if !v.Installed {
log.Tracef("skip %s, not installed", v.Name)
// if the sub depends on a collection that is not a direct or indirect dependency
// of the current item, it is not removed
for _, subParent := range sub.Ancestors() {
if !purge && !subParent.State.Installed {
continue
}
if !v.Downloaded {
log.Warningf("%s : not downloaded, please install.", v.Name)
// the ancestor that would block the removal of the sub item is also an ancestor
// of the item we are removing, so we don't want false warnings
// (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux,
// while we are removing crowdsecurity/sshd)
if slices.Contains(ancestors, subParent) {
continue
}
found = true
// the sub-item belongs to the item we are removing, but we already knew that
if subParent == i {
continue
}
if v.UpToDate {
log.Infof("%s : up-to-date", v.Name)
if !slices.Contains(descendants, subParent) {
log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
continue
}
}
if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil {
log.Fatalf("%s : download failed : %v", v.Name, err)
subRemoved, err := sub.Remove(purge, force)
if err != nil {
return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
}
removed = removed || subRemoved
}
if err = i.disable(purge, force); err != nil {
return false, fmt.Errorf("while removing %s: %w", i.Name, err)
}
removed = true
return removed, nil
}
// Upgrade downloads and applies the last version of the item from the hub.
func (i *Item) Upgrade(force bool) (bool, error) {
updated := false
if !i.State.Downloaded {
return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
}
if !i.State.Installed {
return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name)
}
if i.State.UpToDate {
log.Infof("%s: up-to-date", i.Name)
if err := i.DownloadDataIfNeeded(force); err != nil {
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
}
if !force {
// no upgrade needed
return false, nil
}
}
if _, err := i.downloadLatest(force, true); err != nil {
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
}
if !i.State.UpToDate {
if i.State.Tainted {
log.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
} else if i.IsLocal() {
log.Infof("%v %s is local", emoji.Prohibited, i.Name)
}
} else {
// a check on stdout is used while scripting to know if the hub has been upgraded
// and a configuration reload is required
// TODO: use a better way to communicate this
fmt.Printf("updated %s\n", i.Name)
log.Infof("%v %s: updated", emoji.Package, i.Name)
updated = true
}
return updated, nil
}
// downloadLatest downloads the latest version of the item to the hub directory.
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
log.Debugf("Downloading %s %s", i.Type, i.Name)
for _, sub := range i.SubItems() {
if !sub.State.Installed && updateOnly && sub.State.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name)
continue
}
}
if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil {
log.Fatalf("%s : download failed : %v", v.Name, err)
}
log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly)
if !v.UpToDate {
if v.Tainted {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
} else if v.Local {
log.Infof("%v %s is local", emoji.Prohibited, v.Name)
}
} else {
// this is used while scripting to know if the hub has been upgraded
// and a configuration reload is required
fmt.Printf("updated %s\n", v.Name)
log.Infof("%v %s : updated", emoji.Package, v.Name)
updated++
}
// recurse as it's a collection
if sub.HasSubItems() {
log.Tracef("collection, recurse")
if err := AddItem(itemType, v); err != nil {
log.Fatalf("unable to add %s: %v", v.Name, err)
if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil {
return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
}
}
if !found && name == "" {
log.Infof("No %s installed, nothing to upgrade", itemType)
} else if !found {
log.Errorf("Item '%s' not found in hub", name)
} else if updated == 0 && found {
if name == "" {
log.Infof("All %s are already up-to-date", itemType)
} else {
log.Infof("Item '%s' is up-to-date", name)
downloaded := sub.State.Downloaded
if _, err := sub.download(overwrite); err != nil {
return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
}
} else if updated != 0 {
log.Infof("Upgraded %d items", updated)
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if sub.Downloaded is false because maybe the item has been disabled by the user.
if !sub.State.Installed && !downloaded {
if err := sub.enable(); err != nil {
return "", fmt.Errorf("enabling '%s': %w", sub.Name, err)
}
}
}
if !i.State.Installed && updateOnly && i.State.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name)
return "", nil
}
ret, err := i.download(overwrite)
if err != nil {
return "", fmt.Errorf("failed to download item: %w", err)
}
return ret, nil
}
// fetch downloads the item from the hub, verifies the hash and returns the content.
func (i *Item) fetch() ([]byte, error) {
url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
return nil, fmt.Errorf("failed to build hub item request: %w", err)
}
resp, err := hubClient.Get(url)
if err != nil {
return nil, fmt.Errorf("while downloading %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("while downloading %s: %w", url, err)
}
hash := sha256.New()
if _, err = hash.Write(body); err != nil {
return nil, fmt.Errorf("while hashing %s: %w", i.Name, err)
}
meow := hex.EncodeToString(hash.Sum(nil))
if meow != i.Versions[i.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
return nil, fmt.Errorf("invalid download hash for %s", i.Name)
}
return body, nil
}
// download downloads the item from the hub and writes it to the hub directory.
func (i *Item) download(overwrite bool) (string, error) {
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
if i.State.Tainted {
log.Debugf("%s: tainted, not updated", i.Name)
return "", nil
}
if i.State.UpToDate {
// We still have to check if data files are present
log.Debugf("%s: up-to-date, not updated", i.Name)
}
}
body, err := i.fetch()
if err != nil {
return "", err
}
// all good, install
// ensure that target file is within target dir
finalPath, err := i.downloadPath()
if err != nil {
return "", err
}
parentDir := filepath.Dir(finalPath)
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
return "", fmt.Errorf("while creating %s: %w", parentDir, err)
}
// check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s: overwrite", i.Name)
log.Debugf("target: %s", finalPath)
} else {
log.Infof("%s: OK", i.Name)
}
if err = os.WriteFile(finalPath, body, 0o644); err != nil {
return "", fmt.Errorf("while writing %s: %w", finalPath, err)
}
i.State.Downloaded = true
i.State.Tainted = false
i.State.UpToDate = true
if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
}
return finalPath, nil
}
// DownloadDataIfNeeded downloads the data set for the item.
func (i *Item) DownloadDataIfNeeded(force bool) error {
itemFilePath, err := i.installPath()
if err != nil {
return err
}
itemFile, err := os.Open(itemFilePath)
if err != nil {
return fmt.Errorf("while opening %s: %w", itemFilePath, err)
}
defer itemFile.Close()
if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}
return nil
}

218
pkg/cwhub/helpers_test.go
View file

@ -4,157 +4,187 @@ import (
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
// We expect the new scenario to be installed
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection.
// We expect the new scenario to be installed.
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
getHubIdxOrFail(t)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
// This is the scenario that gets added in next version of collection
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
pushUpdateToCollectionInHub()
if err := UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
getHubIdxOrFail(t)
hub, err := NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
hub = getHubOrFail(t, hub.local, remote)
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.True(t, didUpdate)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
}
// Install a collection, disable a scenario.
// Upgrade should install should not enable/download the disabled scenario.
func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
getHubIdxOrFail(t)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
didRemove, err := item.Remove(false, false)
require.NoError(t, err)
require.True(t, didRemove)
RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
getHubIdxOrFail(t)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
if err := UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
getHubIdxOrFail(t)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
hub, err = NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.False(t, didUpdate)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
}
func getHubIdxOrFail(t *testing.T) {
if err := GetHubIdx(getTestCfg().Hub); err != nil {
t.Fatalf("failed to load hub index")
}
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
hub, err := NewHub(local, remote, false)
require.NoError(t, err, "failed to load hub index")
return hub
}
// Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
// Upgrade should not enable/download the disabled scenario.
// Upgrade should install and enable the newly added scenario.
func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
getHubIdxOrFail(t)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
didRemove, err := item.Remove(false, false)
require.NoError(t, err)
require.True(t, didRemove)
RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
getHubIdxOrFail(t)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
// we just removed. Nor should it install the newly added scenario
pushUpdateToCollectionInHub()
if err := UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err)
}
hub, err = NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
getHubIdxOrFail(t)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
hub = getHubOrFail(t, hub.local, remote)
UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
getHubIdxOrFail(t)
require.False(t, hubIdx[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hubIdx[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.True(t, didUpdate)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
}
func assertCollectionDepsInstalled(t *testing.T, collection string) {
func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
t.Helper()
c := hubIdx[COLLECTIONS][collection]
require.NoError(t, CollecDepsCheck(&c))
c := hub.Items[COLLECTIONS][collection]
require.NoError(t, c.checkSubItemVersions())
}
func pushUpdateToCollectionInHub() {

161
pkg/cwhub/hub.go Normal file
View file

@ -0,0 +1,161 @@
package cwhub
import (
"bytes"
"encoding/json"
"fmt"
"os"
"path"
"strings"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
// Hub is the main structure for the package.
type Hub struct {
Items HubItems // Items read from HubDir and InstallDir
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
Warnings []string // Warnings encountered during sync
}
// GetDataDir returns the data directory, where data sets are installed.
func (h *Hub) GetDataDir() string {
return h.local.InstallDataDir
}
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
// If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) {
if local == nil {
return nil, fmt.Errorf("no hub configuration found")
}
hub := &Hub{
local: local,
remote: remote,
}
if updateIndex {
if err := hub.updateIndex(); err != nil {
return nil, err
}
}
log.Debugf("loading hub idx %s", local.HubIndexFile)
if err := hub.parseIndex(); err != nil {
return nil, fmt.Errorf("failed to load index: %w", err)
}
if err := hub.localSync(); err != nil {
return nil, fmt.Errorf("failed to sync items: %w", err)
}
return hub, nil
}
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections.
func (h *Hub) parseIndex() error {
bidx, err := os.ReadFile(h.local.HubIndexFile)
if err != nil {
return fmt.Errorf("unable to read index file: %w", err)
}
if err := json.Unmarshal(bidx, &h.Items); err != nil {
return fmt.Errorf("failed to unmarshal index: %w", err)
}
log.Debugf("%d item types in hub index", len(ItemTypes))
// Iterate over the different types to complete the struct
for _, itemType := range ItemTypes {
log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
for name, item := range h.Items[itemType] {
item.hub = h
item.Name = name
// if the item has no (redundant) author, take it from the json key
if item.Author == "" && strings.Contains(name, "/") {
item.Author = strings.Split(name, "/")[0]
}
item.Type = itemType
item.FileName = path.Base(item.RemotePath)
item.logMissingSubItems()
}
}
return nil
}
// ItemStats returns total counts of the hub items, including local and tainted.
func (h *Hub) ItemStats() []string {
loaded := ""
local := 0
tainted := 0
for _, itemType := range ItemTypes {
if len(h.Items[itemType]) == 0 {
continue
}
loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
for _, item := range h.Items[itemType] {
if item.IsLocal() {
local++
}
if item.State.Tainted {
tainted++
}
}
}
loaded = strings.Trim(loaded, ", ")
if loaded == "" {
loaded = "0 items"
}
ret := []string{
fmt.Sprintf("Loaded: %s", loaded),
}
if local > 0 || tainted > 0 {
ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", local, tainted))
}
return ret
}
// updateIndex downloads the latest version of the index and writes it to disk if it changed.
func (h *Hub) updateIndex() error {
body, err := h.remote.fetchIndex()
if err != nil {
return err
}
oldContent, err := os.ReadFile(h.local.HubIndexFile)
if err != nil {
if !os.IsNotExist(err) {
log.Warningf("failed to read hub index: %s", err)
}
} else if bytes.Equal(body, oldContent) {
log.Info("hub index is up to date")
return nil
}
if err = os.WriteFile(h.local.HubIndexFile, body, 0o644); err != nil {
return fmt.Errorf("failed to write hub index: %w", err)
}
log.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body))
return nil
}

77
pkg/cwhub/hub_test.go Normal file
View file

@ -0,0 +1,77 @@
package cwhub
import (
"fmt"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestInitHubUpdate(t *testing.T) {
hub := envSetup(t)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
_, err := NewHub(hub.local, remote, true)
require.NoError(t, err)
}
func TestUpdateIndex(t *testing.T) {
// bad url template
fmt.Println("Test 'bad URL'")
tmpIndex, err := os.CreateTemp("", "index.json")
require.NoError(t, err)
t.Cleanup(func() {
os.Remove(tmpIndex.Name())
})
hub := envSetup(t)
hub.remote = &RemoteHubCfg{
URLTemplate: "x",
Branch: "",
IndexPath: "",
}
hub.local.HubIndexFile = tmpIndex.Name()
err = hub.updateIndex()
cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'")
// bad domain
fmt.Println("Test 'bad domain'")
hub.remote = &RemoteHubCfg{
URLTemplate: "https://baddomain/%s/%s",
Branch: "master",
IndexPath: ".index.json",
}
err = hub.updateIndex()
require.NoError(t, err)
// XXX: this is not failing
// cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
// bad target path
fmt.Println("Test 'bad target path'")
hub.remote = &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
hub.local.HubIndexFile = "/does/not/exist/index.json"
err = hub.updateIndex()
cstest.RequireErrorContains(t, err, "failed to write hub index: open /does/not/exist/index.json:")
}

214
pkg/cwhub/install.go
View file

@ -1,214 +0,0 @@
package cwhub
import (
"fmt"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func purgeItem(hub *csconfig.Hub, target Item) (Item, error) {
itempath := hub.HubDir + "/" + target.RemotePath
// disable hub file
if err := os.Remove(itempath); err != nil {
return target, fmt.Errorf("while removing file: %w", err)
}
target.Downloaded = false
log.Infof("Removed source file [%s]: %s", target.Name, itempath)
hubIdx[target.Type][target.Name] = target
return target, nil
}
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
func DisableItem(hub *csconfig.Hub, target *Item, purge bool, force bool) error {
var err error
// already disabled, noop unless purge
if !target.Installed {
if purge {
*target, err = purgeItem(hub, *target)
if err != nil {
return err
}
}
return nil
}
if target.Local {
return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
}
if target.Tainted && !force {
return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
}
// for a COLLECTIONS, disable sub-items
if target.Type == COLLECTIONS {
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if val, ok := hubIdx[ptrtype][p]; ok {
// check if the item doesn't belong to another collection before removing it
toRemove := true
for _, collection := range val.BelongsToCollections {
if collection != target.Name {
toRemove = false
break
}
}
if toRemove {
err = DisableItem(hub, &val, purge, force)
if err != nil {
return fmt.Errorf("while disabling %s: %w", p, err)
}
} else {
log.Infof("%s was not removed because it belongs to another collection", val.Name)
}
} else {
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
}
}
}
}
syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
if err != nil {
return err
}
stat, err := os.Lstat(syml)
if os.IsNotExist(err) {
// we only accept to "delete" non existing items if it's a forced purge
if !purge && !force {
return fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml)
}
} else {
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml)
return fmt.Errorf("%s isn't managed by hub", target.Name)
}
hubpath, err := os.Readlink(syml)
if err != nil {
return fmt.Errorf("while reading symlink: %w", err)
}
absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while abs path: %w", err)
}
if hubpath != absPath {
log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath)
return fmt.Errorf("%s isn't managed by hub", target.Name)
}
// remove the symlink
if err = os.Remove(syml); err != nil {
return fmt.Errorf("while removing symlink: %w", err)
}
log.Infof("Removed symlink [%s] : %s", target.Name, syml)
}
target.Installed = false
if purge {
*target, err = purgeItem(hub, *target)
if err != nil {
return err
}
}
hubIdx[target.Type][target.Name] = *target
return nil
}
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
// Handles collections recursively
func EnableItem(hub *csconfig.Hub, target *Item) error {
var err error
parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
// create directories if needed
if target.Installed {
if target.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
}
if target.Local {
return fmt.Errorf("%s is local, won't enable", target.Name)
}
// if it's a collection, check sub-items even if the collection file itself is up-to-date
if target.UpToDate && target.Type != COLLECTIONS {
log.Tracef("%s is installed and up-to-date, skip.", target.Name)
return nil
}
}
if _, err = os.Stat(parentDir); os.IsNotExist(err) {
log.Infof("%s doesn't exist, create", parentDir)
if err = os.MkdirAll(parentDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating directory: %w", err)
}
}
// install sub-items if it's a collection
if target.Type == COLLECTIONS {
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx[ptrtype][p]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
err = EnableItem(hub, &val)
if err != nil {
return fmt.Errorf("while installing %s: %w", p, err)
}
}
}
}
// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
if _, err = os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) {
log.Infof("%s already exists.", parentDir+"/"+target.FileName)
return nil
}
// hub.ConfigDir + target.RemotePath
srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while getting source path: %w", err)
}
dstPath, err := filepath.Abs(parentDir + "/" + target.FileName)
if err != nil {
return fmt.Errorf("while getting destination path: %w", err)
}
if err = os.Symlink(srcPath, dstPath); err != nil {
return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
}
log.Infof("Enabled %s : %s", target.Type, target.Name)
target.Installed = true
hubIdx[target.Type][target.Name] = *target
return nil
}

383
pkg/cwhub/items.go Normal file
View file

@ -0,0 +1,383 @@
package cwhub
import (
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/Masterminds/semver/v3"
"github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
)
const (
// managed item types.
COLLECTIONS = "collections"
PARSERS = "parsers"
POSTOVERFLOWS = "postoverflows"
SCENARIOS = "scenarios"
)
const (
versionUpToDate = iota // the latest version from index is installed
versionUpdateAvailable // not installed, or lower than latest
versionUnknown // local file with no version, or invalid version number
versionFuture // local version is higher latest, but is included in the index: should not happen
)
var (
// The order is important, as it is used to range over sub-items in collections.
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
)
type HubItems map[string]map[string]*Item
// ItemVersion is used to detect the version of a given item
// by comparing the hash of each version to the local file.
// If the item does not match any known version, it is considered tainted (modified).
type ItemVersion struct {
Digest string `json:"digest,omitempty" yaml:"digest,omitempty"`
Deprecated bool `json:"deprecated,omitempty" yaml:"deprecated,omitempty"`
}
// ItemState is used to keep the local state (i.e. at runtime) of an item.
// This data is not stored in the index, but is displayed with "cscli ... inspect".
type ItemState struct {
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"`
LocalVersion string `json:"local_version,omitempty" yaml:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty" yaml:"local_hash,omitempty"`
Installed bool `json:"installed"`
Downloaded bool `json:"downloaded"`
UpToDate bool `json:"up_to_date"`
Tainted bool `json:"tainted"`
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
}
// Item is created from an index file and enriched with local info.
type Item struct {
hub *Hub // back pointer to the hub, to retrieve other items and call install/remove methods
State ItemState `json:"-" yaml:"-"` // local state, not stored in the index
Type string `json:"type,omitempty" yaml:"type,omitempty"` // one of the ItemTypes
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
Name string `json:"name,omitempty" yaml:"name,omitempty"` // usually "author/name"
FileName string `json:"file_name,omitempty" yaml:"file_name,omitempty"` // eg. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"`
Author string `json:"author,omitempty" yaml:"author,omitempty"`
References []string `json:"references,omitempty" yaml:"references,omitempty"`
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty" yaml:"version,omitempty"` // the last available version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions
// if it's a collection, it can have sub items
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
func (i *Item) HasSubItems() bool {
return i.Type == COLLECTIONS
}
// IsLocal returns true if the item has been create by a user (not downloaded from the hub).
func (i *Item) IsLocal() bool {
return i.State.Installed && !i.State.Downloaded
}
// MarshalJSON is used to prepare the output for "cscli ... inspect -o json".
// It must not use a pointer receiver.
func (i Item) MarshalJSON() ([]byte, error) {
type Alias Item
return json.Marshal(&struct {
Alias
// we have to repeat the fields here, json will have inline support in v2
LocalPath string `json:"local_path,omitempty"`
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"`
Installed bool `json:"installed"`
Downloaded bool `json:"downloaded"`
UpToDate bool `json:"up_to_date"`
Tainted bool `json:"tainted"`
Local bool `json:"local"`
BelongsToCollections []string `json:"belongs_to_collections,omitempty"`
}{
Alias: Alias(i),
LocalPath: i.State.LocalPath,
LocalVersion: i.State.LocalVersion,
LocalHash: i.State.LocalHash,
Installed: i.State.Installed,
Downloaded: i.State.Downloaded,
UpToDate: i.State.UpToDate,
Tainted: i.State.Tainted,
BelongsToCollections: i.State.BelongsToCollections,
Local: i.IsLocal(),
})
}
// MarshalYAML is used to prepare the output for "cscli ... inspect -o raw".
// It must not use a pointer receiver.
func (i Item) MarshalYAML() (interface{}, error) {
type Alias Item
return &struct {
Alias `yaml:",inline"`
State ItemState `yaml:",inline"`
Local bool `yaml:"local"`
}{
Alias: Alias(i),
State: i.State,
Local: i.IsLocal(),
}, nil
}
// SubItems returns a slice of sub-items, excluding the ones that were not found.
func (i *Item) SubItems() []*Item {
sub := make([]*Item, 0)
for _, name := range i.Parsers {
s := i.hub.GetItem(PARSERS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.PostOverflows {
s := i.hub.GetItem(POSTOVERFLOWS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.Scenarios {
s := i.hub.GetItem(SCENARIOS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.Collections {
s := i.hub.GetItem(COLLECTIONS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
return sub
}
func (i *Item) logMissingSubItems() {
if !i.HasSubItems() {
return
}
for _, subName := range i.Parsers {
if i.hub.GetItem(PARSERS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name)
}
}
for _, subName := range i.Scenarios {
if i.hub.GetItem(SCENARIOS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name)
}
}
for _, subName := range i.PostOverflows {
if i.hub.GetItem(POSTOVERFLOWS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name)
}
}
for _, subName := range i.Collections {
if i.hub.GetItem(COLLECTIONS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name)
}
}
}
// Ancestors returns a slice of items (typically collections) that have this item as a direct or indirect dependency.
func (i *Item) Ancestors() []*Item {
ret := make([]*Item, 0)
for _, parentName := range i.State.BelongsToCollections {
parent := i.hub.GetItem(COLLECTIONS, parentName)
if parent == nil {
continue
}
ret = append(ret, parent)
}
return ret
}
// InstallStatus returns the status of the item as a string and an emoji
// (eg. "enabled,update-available" and emoji.Warning).
func (i *Item) InstallStatus() (string, emoji.Emoji) {
status := "disabled"
ok := false
if i.State.Installed {
ok = true
status = "enabled"
}
managed := true
if i.IsLocal() {
managed = false
status += ",local"
}
warning := false
if i.State.Tainted {
warning = true
status += ",tainted"
} else if !i.State.UpToDate && !i.IsLocal() {
warning = true
status += ",update-available"
}
emo := emoji.QuestionMark
switch {
case !managed:
emo = emoji.House
case !i.State.Installed:
emo = emoji.Prohibited
case warning:
emo = emoji.Warning
case ok:
emo = emoji.CheckMark
}
return status, emo
}
// versionStatus returns the status of the item version compared to the hub version.
// semver requires the 'v' prefix.
func (i *Item) versionStatus() int {
local, err := semver.NewVersion(i.State.LocalVersion)
if err != nil {
return versionUnknown
}
// hub versions are already validated while syncing, ignore errors
latest, _ := semver.NewVersion(i.Version)
if local.LessThan(latest) {
return versionUpdateAvailable
}
if local.Equal(latest) {
return versionUpToDate
}
return versionFuture
}
// validPath returns true if the (relative) path is allowed for the item.
// dirNname: the directory name (ie. crowdsecurity).
// fileName: the filename (ie. apache2-logs.yaml).
func (i *Item) validPath(dirName, fileName string) bool {
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
}
// GetItemMap returns the map of items for a given type.
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
return h.Items[itemType]
}
// GetItem returns an item from hub based on its type and full name (author/name).
func (h *Hub) GetItem(itemType string, itemName string) *Item {
return h.GetItemMap(itemType)[itemName]
}
// GetItemNames returns a slice of (full) item names for a given type
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
func (h *Hub) GetItemNames(itemType string) []string {
m := h.GetItemMap(itemType)
if m == nil {
return nil
}
names := make([]string, 0, len(m))
for k := range m {
names = append(names, k)
}
return names
}
// GetAllItems returns a slice of all the items of a given type, installed or not.
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
ret := make([]*Item, len(items))
idx := 0
for _, item := range items {
ret[idx] = item
idx++
}
return ret, nil
}
// GetInstalledItems returns a slice of the installed items of a given type.
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
retItems := make([]*Item, 0)
for _, item := range items {
if item.State.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
// GetInstalledItemNames returns the names of the installed items of a given type.
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
items, err := h.GetInstalledItems(itemType)
if err != nil {
return nil, err
}
retStr := make([]string, len(items))
for idx, it := range items {
retStr[idx] = it.Name
}
return retStr, nil
}
// SortItemSlice sorts a slice of items by name, case insensitive.
func SortItemSlice(items []*Item) {
sort.Slice(items, func(i, j int) bool {
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
})
}

71
pkg/cwhub/items_test.go Normal file
View file

@ -0,0 +1,71 @@
package cwhub
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestItemStatus(t *testing.T) {
hub := envSetup(t)
// get existing map
x := hub.GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item: good and bad
for k := range x {
item := hub.GetItem(COLLECTIONS, k)
require.NotNil(t, item)
item.State.Installed = true
item.State.UpToDate = false
item.State.Tainted = false
item.State.Downloaded = true
txt, _ := item.InstallStatus()
require.Equal(t, "enabled,update-available", txt)
item.State.Installed = true
item.State.UpToDate = false
item.State.Tainted = false
item.State.Downloaded = false
txt, _ = item.InstallStatus()
require.Equal(t, "enabled,local", txt)
}
stats := hub.ItemStats()
require.Equal(t, []string{
"Loaded: 2 parsers, 1 scenarios, 3 collections",
"Unmanaged items: 3 local, 0 tainted",
}, stats)
}
func TestGetters(t *testing.T) {
hub := envSetup(t)
// get non existing map
empty := hub.GetItemMap("ratata")
require.Nil(t, empty)
// get existing map
x := hub.GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item: good and bad
for k := range x {
empty := hub.GetItem(COLLECTIONS, k+"nope")
require.Nil(t, empty)
item := hub.GetItem(COLLECTIONS, k)
require.NotNil(t, item)
// Add item and get it
item.Name += "nope"
hub.Items[item.Type][item.Name] = item
newitem := hub.GetItem(COLLECTIONS, item.Name)
require.NotNil(t, newitem)
}
}

53
pkg/cwhub/leakybucket.go Normal file
View file

@ -0,0 +1,53 @@
package cwhub
// Resolve a symlink to find the hub item it points to.
// This file is used only by pkg/leakybucket
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary.
func itemKey(itemPath string) (string, error) {
f, err := os.Lstat(itemPath)
if err != nil {
return "", fmt.Errorf("while performing lstat on %s: %w", itemPath, err)
}
if f.Mode()&os.ModeSymlink == 0 {
// it's not a symlink, so the filename itsef should be the key
return filepath.Base(itemPath), nil
}
// resolve the symlink to hub file
pathInHub, err := os.Readlink(itemPath)
if err != nil {
return "", fmt.Errorf("while reading symlink of %s: %w", itemPath, err)
}
author := filepath.Base(filepath.Dir(pathInHub))
fname := filepath.Base(pathInHub)
fname = strings.TrimSuffix(fname, ".yaml")
fname = strings.TrimSuffix(fname, ".yml")
return fmt.Sprintf("%s/%s", author, fname), nil
}
// GetItemByPath retrieves an item from the hub index based on its local path.
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
itemKey, err := itemKey(itemPath)
if err != nil {
return nil, err
}
item := h.GetItem(itemType, itemKey)
if item == nil {
return nil, fmt.Errorf("%s not found in %s", itemKey, itemType)
}
return item, nil
}

552
pkg/cwhub/loader.go
View file

@ -1,552 +0,0 @@
package cwhub
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func isYAMLFileName(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
func validItemFileName(vname string, fauthor string, fname string) bool {
return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml")
}
func handleSymlink(path string) (string, error) {
hubpath, err := os.Readlink(path)
if err != nil {
return "", fmt.Errorf("unable to read symlink of %s", path)
}
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
_, err = os.Lstat(hubpath)
if os.IsNotExist(err) {
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
// remove the symlink
if err = os.Remove(path); err != nil {
return "", fmt.Errorf("failed to unlink %s: %w", path, err)
}
// XXX: is this correct?
return "", nil
}
return hubpath, nil
}
func getSHA256(filepath string) (string, error) {
f, err := os.Open(filepath)
if err != nil {
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
type Walker struct {
// the walk/parserVisit function can't receive extra args
hubdir string
installdir string
}
func NewWalker(hub *csconfig.Hub) Walker {
return Walker{
hubdir: hub.HubDir,
installdir: hub.InstallDir,
}
}
type itemFileInfo struct {
fname string
stage string
ftype string
fauthor string
}
func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
ret := itemFileInfo{}
inhub := false
subs := strings.Split(path, string(os.PathSeparator))
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir)
log.Tracef("subs:%v", subs)
// we're in hub (~/.hub/hub/)
if strings.HasPrefix(path, w.hubdir) {
log.Tracef("in hub dir")
inhub = true
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml
//.../hub/profiles/crowdsec/linux.yaml
if len(subs) < 4 {
log.Fatalf("path is too short : %s (%d)", path, len(subs))
}
ret.fname = subs[len(subs)-1]
ret.fauthor = subs[len(subs)-2]
ret.stage = subs[len(subs)-3]
ret.ftype = subs[len(subs)-4]
} else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir")
if len(subs) < 3 {
log.Fatalf("path is too short : %s (%d)", path, len(subs))
}
///.../config/parser/stage/file.yaml
///.../config/postoverflow/stage/file.yaml
///.../config/scenarios/scenar.yaml
///.../config/collections/linux.yaml //file is empty
ret.fname = subs[len(subs)-1]
ret.stage = subs[len(subs)-2]
ret.ftype = subs[len(subs)-3]
ret.fauthor = ""
} else {
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir)
}
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
if ret.stage == SCENARIOS {
ret.ftype = SCENARIOS
ret.stage = ""
} else if ret.stage == COLLECTIONS {
ret.ftype = COLLECTIONS
ret.stage = ""
} else if ret.ftype != PARSERS && ret.ftype != PARSERS_OVFLW {
// its a PARSER / PARSER_OVFLW with a stage
return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
}
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
return ret, inhub, nil
}
func (w Walker) parserVisit(path string, f os.DirEntry, err error) error {
var (
local bool
hubpath string
)
if err != nil {
log.Debugf("while syncing hub dir: %s", err)
// there is a path error, we ignore the file
return nil
}
path, err = filepath.Abs(path)
if err != nil {
return err
}
// we only care about files
if f == nil || f.IsDir() {
return nil
}
if !isYAMLFileName(f.Name()) {
return nil
}
info, inhub, err := w.getItemInfo(path)
if err != nil {
return err
}
/*
we can encounter 'collections' in the form of a symlink :
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
when the collection is installed, both files are created
*/
// non symlinks are local user files or hub files
if f.Type()&os.ModeSymlink == 0 {
local = true
log.Tracef("%s isn't a symlink", path)
} else {
hubpath, err = handleSymlink(path)
if err != nil {
return err
}
log.Tracef("%s points to %s", path, hubpath)
if hubpath == "" {
// XXX: is this correct?
return nil
}
}
// if it's not a symlink and not in hub, it's a local file, don't bother
if local && !inhub {
log.Tracef("%s is a local file, skip", path)
skippedLocal++
// log.Infof("local scenario, skip.")
_, fileName := filepath.Split(path)
hubIdx[info.ftype][info.fname] = Item{
Name: info.fname,
Stage: info.stage,
Installed: true,
Type: info.ftype,
Local: true,
LocalPath: path,
UpToDate: true,
FileName: fileName,
}
return nil
}
// try to find which configuration item it is
log.Tracef("check [%s] of %s", info.fname, info.ftype)
match := false
for name, item := range hubIdx[info.ftype] {
log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
if info.fname != item.FileName {
log.Tracef("%s != %s (filename)", info.fname, item.FileName)
continue
}
// wrong stage
if item.Stage != info.stage {
continue
}
// if we are walking hub dir, just mark present files as downloaded
if inhub {
// wrong author
if info.fauthor != item.Author {
continue
}
// wrong file
if !validItemFileName(item.Name, info.fauthor, info.fname) {
continue
}
if path == w.hubdir+"/"+item.RemotePath {
log.Tracef("marking %s as downloaded", item.Name)
item.Downloaded = true
}
} else if !hasPathSuffix(hubpath, item.RemotePath) {
// wrong file
// <type>/<stage>/<author>/<name>.yaml
continue
}
sha, err := getSHA256(path)
if err != nil {
log.Fatalf("Failed to get sha of %s : %v", path, err)
}
// let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(item.Versions))
for k := range item.Versions {
versions = append(versions, k)
}
sort.Sort(sort.Reverse(sort.StringSlice(versions)))
for _, version := range versions {
val := item.Versions[version]
if sha != val.Digest {
// log.Infof("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
continue
}
// we got an exact match, update struct
item.Downloaded = true
item.LocalHash = sha
if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version)
item.LocalPath = path
item.LocalVersion = version
item.Tainted = false
// if we're walking the hub, present file doesn't means installed file
item.Installed = true
}
if version == item.Version {
log.Tracef("%s is up-to-date", item.Name)
item.UpToDate = true
}
match = true
break
}
if !match {
log.Tracef("got tainted match for %s: %s", item.Name, path)
skippedTainted++
// the file and the stage is right, but the hash is wrong, it has been tainted by user
if !inhub {
item.LocalPath = path
item.Installed = true
}
item.UpToDate = false
item.LocalVersion = "?"
item.Tainted = true
item.LocalHash = sha
}
// update the entry if appropriate
// if _, ok := hubIdx[ftype][k]; !ok || !inhub || v.D {
// fmt.Printf("Updating %s", k)
// hubIdx[ftype][k] = v
// } else if !inhub {
// } else if
hubIdx[info.ftype][name] = item
return nil
}
log.Infof("Ignoring file %s of type %s", path, info.ftype)
return nil
}
func CollecDepsCheck(v *Item) error {
if v.versionStatus() != 0 { // not up-to-date
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
return nil
}
if v.Type != COLLECTIONS {
return nil
}
// if it's a collection, ensure all the items are installed, or tag it as tainted
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
sliceType := ItemTypes[idx]
for _, subName := range itemSlice {
subItem, ok := hubIdx[sliceType][subName]
if !ok {
log.Fatalf("Referred %s %s in collection %s doesn't exist.", sliceType, subName, v.Name)
}
log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
if !v.Installed {
continue
}
if subItem.Type == COLLECTIONS {
log.Tracef("collec, recurse.")
if err := CollecDepsCheck(&subItem); err != nil {
if subItem.Tainted {
v.Tainted = true
}
return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
}
hubIdx[sliceType][subName] = subItem
}
// propagate the state of sub-items to set
if subItem.Tainted {
v.Tainted = true
return fmt.Errorf("tainted %s %s, tainted", sliceType, subName)
}
if !subItem.Installed && v.Installed {
v.Tainted = true
return fmt.Errorf("missing %s %s, tainted", sliceType, subName)
}
if !subItem.UpToDate {
v.UpToDate = false
return fmt.Errorf("outdated %s %s", sliceType, subName)
}
skip := false
for idx := range subItem.BelongsToCollections {
if subItem.BelongsToCollections[idx] == v.Name {
skip = true
}
}
if !skip {
subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
}
hubIdx[sliceType][subName] = subItem
log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
}
}
return nil
}
func SyncDir(hub *csconfig.Hub, dir string) ([]string, error) {
warnings := []string{}
// For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last
for _, scan := range ItemTypes {
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
if err != nil {
log.Errorf("failed %s : %s", cpath, err)
}
err = filepath.WalkDir(cpath, NewWalker(hub).parserVisit)
if err != nil {
return warnings, err
}
}
for name, item := range hubIdx[COLLECTIONS] {
if !item.Installed {
continue
}
vs := item.versionStatus()
switch vs {
case 0: // latest
if err := CollecDepsCheck(&item); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
hubIdx[COLLECTIONS][name] = item
}
case 1: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
default: // version is higher than the highest available from hub?
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
}
log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
}
return warnings, nil
}
// Updates the info from HubInit() with the local state
func LocalSync(hub *csconfig.Hub) ([]string, error) {
skippedLocal = 0
skippedTainted = 0
warnings, err := SyncDir(hub, hub.InstallDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err)
}
_, err = SyncDir(hub, hub.HubDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err)
}
return warnings, nil
}
func GetHubIdx(hub *csconfig.Hub) error {
if hub == nil {
return fmt.Errorf("no configuration found for hub")
}
log.Debugf("loading hub idx %s", hub.HubIndexFile)
bidx, err := os.ReadFile(hub.HubIndexFile)
if err != nil {
return fmt.Errorf("unable to read index file: %w", err)
}
ret, err := LoadPkgIndex(bidx)
if err != nil {
if !errors.Is(err, ErrMissingReference) {
return fmt.Errorf("unable to load existing index: %w", err)
}
// XXX: why the error check if we bail out anyway?
return err
}
hubIdx = ret
_, err = LocalSync(hub)
if err != nil {
return fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
}
return nil
}
// LoadPkgIndex loads a local .index.json file and returns the map of associated parsers/scenarios/collections
func LoadPkgIndex(buff []byte) (map[string]map[string]Item, error) {
var (
RawIndex map[string]map[string]Item
missingItems []string
)
if err := json.Unmarshal(buff, &RawIndex); err != nil {
return nil, fmt.Errorf("failed to unmarshal index: %w", err)
}
log.Debugf("%d item types in hub index", len(ItemTypes))
// Iterate over the different types to complete the struct
for _, itemType := range ItemTypes {
log.Tracef("%d item", len(RawIndex[itemType]))
for name, item := range RawIndex[itemType] {
item.Name = name
item.Type = itemType
x := strings.Split(item.RemotePath, "/")
item.FileName = x[len(x)-1]
RawIndex[itemType][name] = item
if itemType != COLLECTIONS {
continue
}
// if it's a collection, check its sub-items are present
// XXX should be done later
for idx, ptr := range [][]string{item.Parsers, item.PostOverflows, item.Scenarios, item.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if _, ok := RawIndex[ptrtype][p]; !ok {
log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, item.Name)
missingItems = append(missingItems, p)
}
}
}
}
}
if len(missingItems) > 0 {
return RawIndex, fmt.Errorf("%q: %w", missingItems, ErrMissingReference)
}
return RawIndex, nil
}

61
pkg/cwhub/remote.go Normal file
View file

@ -0,0 +1,61 @@
package cwhub
import (
"fmt"
"io"
"net/http"
)
// RemoteHubCfg is used to retrieve index and items from the remote hub.
type RemoteHubCfg struct {
Branch string
URLTemplate string
IndexPath string
}
// urlTo builds the URL to download a file from the remote hub.
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
if r == nil {
return "", ErrNilRemoteHub
}
// the template must contain two string placeholders
if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
}
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
}
// fetchIndex downloads the index from the hub and returns the content.
func (r *RemoteHubCfg) fetchIndex() ([]byte, error) {
if r == nil {
return nil, ErrNilRemoteHub
}
url, err := r.urlTo(r.IndexPath)
if err != nil {
return nil, fmt.Errorf("failed to build hub index request: %w", err)
}
resp, err := hubClient.Get(url)
if err != nil {
return nil, fmt.Errorf("failed http request for hub index: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
return nil, IndexNotFoundError{url, r.Branch}
}
return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
}
return body, nil
}

498
pkg/cwhub/sync.go Normal file
View file

@ -0,0 +1,498 @@
package cwhub
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"github.com/Masterminds/semver/v3"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)
func isYAMLFileName(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
// linkTarget returns the target of a symlink, or empty string if it's dangling.
func linkTarget(path string) (string, error) {
hubpath, err := os.Readlink(path)
if err != nil {
return "", fmt.Errorf("unable to read symlink: %s", path)
}
log.Tracef("symlink %s -> %s", path, hubpath)
_, err = os.Lstat(hubpath)
if os.IsNotExist(err) {
log.Infof("link target does not exist: %s -> %s", path, hubpath)
return "", nil
}
return hubpath, nil
}
func getSHA256(filepath string) (string, error) {
f, err := os.Open(filepath)
if err != nil {
return "", fmt.Errorf("unable to open '%s': %w", filepath, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// information used to create a new Item, from a file path.
type itemFileInfo struct {
inhub bool
fname string
stage string
ftype string
fauthor string
}
func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
var ret *itemFileInfo
hubDir := h.local.HubDir
installDir := h.local.InstallDir
subs := strings.Split(path, string(os.PathSeparator))
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
log.Tracef("subs:%v", subs)
// we're in hub (~/.hub/hub/)
if strings.HasPrefix(path, hubDir) {
log.Tracef("in hub dir")
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml
//.../hub/profiles/crowdsec/linux.yaml
if len(subs) < 4 {
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
}
ret = &itemFileInfo{
inhub: true,
fname: subs[len(subs)-1],
fauthor: subs[len(subs)-2],
stage: subs[len(subs)-3],
ftype: subs[len(subs)-4],
}
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir")
if len(subs) < 3 {
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
}
///.../config/parser/stage/file.yaml
///.../config/postoverflow/stage/file.yaml
///.../config/scenarios/scenar.yaml
///.../config/collections/linux.yaml //file is empty
ret = &itemFileInfo{
inhub: false,
fname: subs[len(subs)-1],
stage: subs[len(subs)-2],
ftype: subs[len(subs)-3],
fauthor: "",
}
} else {
return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
}
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
if ret.stage == SCENARIOS {
ret.ftype = SCENARIOS
ret.stage = ""
} else if ret.stage == COLLECTIONS {
ret.ftype = COLLECTIONS
ret.stage = ""
} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
// it's a PARSER / POSTOVERFLOW with a stage
return nil, fmt.Errorf("unknown configuration type for file '%s'", path)
}
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
return ret, nil
}
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver.
func sortedVersions(raw []string) ([]string, error) {
vs := make([]*semver.Version, len(raw))
for idx, r := range raw {
v, err := semver.NewVersion(r)
if err != nil {
return nil, fmt.Errorf("%s: %w", r, err)
}
vs[idx] = v
}
sort.Sort(sort.Reverse(semver.Collection(vs)))
ret := make([]string, len(vs))
for idx, v := range vs {
ret[idx] = v.Original()
}
return ret, nil
}
func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) {
type localItemName struct {
Name string `yaml:"name"`
}
_, fileName := filepath.Split(path)
item := &Item{
hub: h,
Name: info.fname,
Stage: info.stage,
Type: info.ftype,
FileName: fileName,
State: ItemState{
LocalPath: path,
Installed: true,
UpToDate: true,
},
}
// try to read the name from the file
itemName := localItemName{}
itemContent, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read %s: %w", path, err)
}
err = yaml.Unmarshal(itemContent, &itemName)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err)
}
if itemName.Name != "" {
item.Name = itemName.Name
}
return item, nil
}
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
hubpath := ""
if err != nil {
log.Debugf("while syncing hub dir: %s", err)
// there is a path error, we ignore the file
return nil
}
// only happens if the current working directory was removed (!)
path, err = filepath.Abs(path)
if err != nil {
return err
}
// we only care about YAML files
if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) {
return nil
}
info, err := h.getItemFileInfo(path)
if err != nil {
return err
}
// non symlinks are local user files or hub files
if f.Type()&os.ModeSymlink == 0 {
log.Tracef("%s is not a symlink", path)
if !info.inhub {
log.Tracef("%s is a local file, skip", path)
item, err := newLocalItem(h, path, info)
if err != nil {
return err
}
h.Items[info.ftype][item.Name] = item
return nil
}
} else {
hubpath, err = linkTarget(path)
if err != nil {
return err
}
if hubpath == "" {
// target does not exist, the user might have removed the file
// or switched to a hub branch without it
return nil
}
}
// try to find which configuration item it is
log.Tracef("check [%s] of %s", info.fname, info.ftype)
for name, item := range h.Items[info.ftype] {
if info.fname != item.FileName {
continue
}
if item.Stage != info.stage {
continue
}
// if we are walking hub dir, just mark present files as downloaded
if info.inhub {
// wrong author
if info.fauthor != item.Author {
continue
}
// not the item we're looking for
if !item.validPath(info.fauthor, info.fname) {
continue
}
src, err := item.downloadPath()
if err != nil {
return err
}
if path == src {
log.Tracef("marking %s as downloaded", item.Name)
item.State.Downloaded = true
}
} else if !hasPathSuffix(hubpath, item.RemotePath) {
// wrong file
// <type>/<stage>/<author>/<name>.yaml
continue
}
err := item.setVersionState(path, info.inhub)
if err != nil {
return err
}
h.Items[info.ftype][name] = item
return nil
}
log.Infof("Ignoring file %s of type %s", path, info.ftype)
return nil
}
// checkSubItemVersions checks for the presence, taint and version state of sub-items.
func (i *Item) checkSubItemVersions() error {
if !i.HasSubItems() {
return nil
}
if i.versionStatus() != versionUpToDate {
log.Debugf("%s dependencies not checked: not up-to-date", i.Name)
return nil
}
// ensure all the sub-items are installed, or tag the parent as tainted
log.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed)
for _, sub := range i.SubItems() {
log.Tracef("check %s installed:%t", sub.Name, sub.State.Installed)
if !i.State.Installed {
continue
}
if err := sub.checkSubItemVersions(); err != nil {
if sub.State.Tainted {
i.State.Tainted = true
}
return fmt.Errorf("dependency of %s: sub collection %s is broken: %w", i.Name, sub.Name, err)
}
if sub.State.Tainted {
i.State.Tainted = true
return fmt.Errorf("%s is tainted because %s:%s is tainted", i.Name, sub.Type, sub.Name)
}
if !sub.State.Installed && i.State.Installed {
i.State.Tainted = true
return fmt.Errorf("%s is tainted because %s:%s is missing", i.Name, sub.Type, sub.Name)
}
if !sub.State.UpToDate {
i.State.UpToDate = false
return fmt.Errorf("dependency of %s: outdated %s:%s", i.Name, sub.Type, sub.Name)
}
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate)
}
return nil
}
// syncDir scans a directory for items, and updates the Hub state accordingly.
func (h *Hub) syncDir(dir string) error {
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
for _, scan := range ItemTypes {
// cpath: top-level item directory, either downloaded or installed items.
// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
if err != nil {
log.Errorf("failed %s: %s", cpath, err)
continue
}
// explicit check for non existing directory, avoid spamming log.Debug
if _, err = os.Stat(cpath); os.IsNotExist(err) {
log.Tracef("directory %s doesn't exist, skipping", cpath)
continue
}
if err = filepath.WalkDir(cpath, h.itemVisit); err != nil {
return err
}
}
return nil
}
// insert a string in a sorted slice, case insensitive, and return the new slice.
func insertInOrderNoCase(sl []string, value string) []string {
i := sort.Search(len(sl), func(i int) bool {
return strings.ToLower(sl[i]) >= strings.ToLower(value)
})
return append(sl[:i], append([]string{value}, sl[i:]...)...)
}
// localSync updates the hub state with downloaded, installed and local items.
func (h *Hub) localSync() error {
err := h.syncDir(h.local.InstallDir)
if err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
}
if err = h.syncDir(h.local.HubDir); err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
}
warnings := make([]string, 0)
for _, item := range h.Items[COLLECTIONS] {
// check for cyclic dependencies
subs, err := item.descendants()
if err != nil {
return err
}
// populate the sub- and sub-sub-items with the collections they belong to
for _, sub := range subs {
sub.State.BelongsToCollections = insertInOrderNoCase(sub.State.BelongsToCollections, item.Name)
}
if !item.State.Installed {
continue
}
vs := item.versionStatus()
switch vs {
case versionUpToDate: // latest
if err := item.checkSubItemVersions(); err != nil {
warnings = append(warnings, err.Error())
}
case versionUpdateAvailable: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
case versionFuture:
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
case versionUnknown:
if !item.IsLocal() {
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
}
}
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
}
h.Warnings = warnings
return nil
}
func (i *Item) setVersionState(path string, inhub bool) error {
var err error
i.State.LocalHash, err = getSHA256(path)
if err != nil {
return fmt.Errorf("failed to get sha256 of %s: %w", path, err)
}
// let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(i.Versions))
for k := range i.Versions {
versions = append(versions, k)
}
versions, err = sortedVersions(versions)
if err != nil {
return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err)
}
i.State.LocalVersion = "?"
for _, version := range versions {
if i.Versions[version].Digest == i.State.LocalHash {
i.State.LocalVersion = version
break
}
}
if i.State.LocalVersion == "?" {
log.Tracef("got tainted match for %s: %s", i.Name, path)
if !inhub {
i.State.LocalPath = path
i.State.Installed = true
}
i.State.UpToDate = false
i.State.Tainted = true
return nil
}
// we got an exact match, update struct
i.State.Downloaded = true
if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version)
i.State.LocalPath = path
i.State.Tainted = false
// if we're walking the hub, present file doesn't means installed file
i.State.Installed = true
}
if i.State.LocalVersion == i.Version {
log.Tracef("%s is up-to-date", i.Name)
i.State.UpToDate = true
}
return nil
}

147
pkg/hubtest/coverage.go
View file

@ -5,173 +5,194 @@ import (
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type ParserCoverage struct {
Parser string
type Coverage struct {
Name string
TestsCount int
PresentIn map[string]bool //poorman's set
}
type ScenarioCoverage struct {
Scenario string
TestsCount int
PresentIn map[string]bool
}
func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
return nil, fmt.Errorf("no parsers in hub index")
}
func (h *HubTest) GetParsersCoverage() ([]ParserCoverage, error) {
var coverage []ParserCoverage
if _, ok := h.HubIndex.Data[cwhub.PARSERS]; !ok {
return coverage, fmt.Errorf("no parsers in hub index")
}
//populate from hub, iterate in alphabetical order
var pkeys []string
for pname := range h.HubIndex.Data[cwhub.PARSERS] {
pkeys = append(pkeys, pname)
}
sort.Strings(pkeys)
for _, pname := range pkeys {
coverage = append(coverage, ParserCoverage{
Parser: pname,
// populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS])
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {
coverage[i] = Coverage{
Name: name,
TestsCount: 0,
PresentIn: make(map[string]bool),
})
}
}
//parser the expressions a-la-oneagain
// parser the expressions a-la-oneagain
passerts, err := filepath.Glob(".tests/*/parser.assert")
if err != nil {
return coverage, fmt.Errorf("while find parser asserts : %s", err)
return nil, fmt.Errorf("while find parser asserts : %s", err)
}
for _, assert := range passerts {
file, err := os.Open(assert)
if err != nil {
return coverage, fmt.Errorf("while reading %s : %s", assert, err)
return nil, fmt.Errorf("while reading %s : %s", assert, err)
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
assertLine := regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
line := scanner.Text()
log.Debugf("assert line : %s", line)
match := assertLine.FindStringSubmatch(line)
match := parserResultRE.FindStringSubmatch(line)
if len(match) == 0 {
log.Debugf("%s doesn't match", line)
continue
}
sidx := assertLine.SubexpIndex("parser")
sidx := parserResultRE.SubexpIndex("parser")
capturedParser := match[sidx]
for idx, pcover := range coverage {
if pcover.Parser == capturedParser {
if pcover.Name == capturedParser {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
parserNameSplit := strings.Split(pcover.Parser, "/")
parserNameSplit := strings.Split(pcover.Name, "/")
parserNameOnly := parserNameSplit[len(parserNameSplit)-1]
if parserNameOnly == capturedParser {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
capturedParserSplit := strings.Split(capturedParser, "/")
capturedParserName := capturedParserSplit[len(capturedParserSplit)-1]
if capturedParserName == parserNameOnly {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
if capturedParserName == parserNameOnly+"-logs" {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
}
}
file.Close()
}
return coverage, nil
}
func (h *HubTest) GetScenariosCoverage() ([]ScenarioCoverage, error) {
var coverage []ScenarioCoverage
if _, ok := h.HubIndex.Data[cwhub.SCENARIOS]; !ok {
return coverage, fmt.Errorf("no scenarios in hub index")
}
//populate from hub, iterate in alphabetical order
var pkeys []string
for scenarioName := range h.HubIndex.Data[cwhub.SCENARIOS] {
pkeys = append(pkeys, scenarioName)
}
sort.Strings(pkeys)
for _, scenarioName := range pkeys {
coverage = append(coverage, ScenarioCoverage{
Scenario: scenarioName,
TestsCount: 0,
PresentIn: make(map[string]bool),
})
func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
return nil, fmt.Errorf("no scenarios in hub index")
}
//parser the expressions a-la-oneagain
// populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS])
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {
coverage[i] = Coverage{
Name: name,
TestsCount: 0,
PresentIn: make(map[string]bool),
}
}
// parser the expressions a-la-oneagain
passerts, err := filepath.Glob(".tests/*/scenario.assert")
if err != nil {
return coverage, fmt.Errorf("while find scenario asserts : %s", err)
return nil, fmt.Errorf("while find scenario asserts : %s", err)
}
for _, assert := range passerts {
file, err := os.Open(assert)
if err != nil {
return coverage, fmt.Errorf("while reading %s : %s", assert, err)
return nil, fmt.Errorf("while reading %s : %s", assert, err)
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
assertLine := regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
line := scanner.Text()
log.Debugf("assert line : %s", line)
match := assertLine.FindStringSubmatch(line)
match := scenarioResultRE.FindStringSubmatch(line)
if len(match) == 0 {
log.Debugf("%s doesn't match", line)
continue
}
sidx := assertLine.SubexpIndex("scenario")
scanner_name := match[sidx]
sidx := scenarioResultRE.SubexpIndex("scenario")
scannerName := match[sidx]
for idx, pcover := range coverage {
if pcover.Scenario == scanner_name {
if pcover.Name == scannerName {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
scenarioNameSplit := strings.Split(pcover.Scenario, "/")
scenarioNameSplit := strings.Split(pcover.Name, "/")
scenarioNameOnly := scenarioNameSplit[len(scenarioNameSplit)-1]
if scenarioNameOnly == scanner_name {
if scenarioNameOnly == scannerName {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
fixedProbingWord := strings.ReplaceAll(pcover.Scenario, "probbing", "probing")
fixedProbingAssert := strings.ReplaceAll(scanner_name, "probbing", "probing")
fixedProbingWord := strings.ReplaceAll(pcover.Name, "probbing", "probing")
fixedProbingAssert := strings.ReplaceAll(scannerName, "probbing", "probing")
if fixedProbingWord == fixedProbingAssert {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
if fmt.Sprintf("%s-detection", pcover.Scenario) == scanner_name {
if fmt.Sprintf("%s-detection", pcover.Name) == scannerName {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
if fmt.Sprintf("%s-detection", fixedProbingWord) == fixedProbingAssert {
coverage[idx].TestsCount++
coverage[idx].PresentIn[assert] = true
continue
}
}
}
file.Close()
}
return coverage, nil
}

38
pkg/hubtest/hubtest.go
View file

@ -6,6 +6,7 @@ import (
"os/exec"
"path/filepath"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -18,7 +19,7 @@ type HubTest struct {
TemplateConfigPath string
TemplateProfilePath string
TemplateSimulationPath string
HubIndex *HubIndex
HubIndex *cwhub.Hub
Tests []*HubTestItem
}
@ -29,42 +30,44 @@ const (
)
func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, error) {
var err error
hubPath, err = filepath.Abs(hubPath)
hubPath, err := filepath.Abs(hubPath)
if err != nil {
return HubTest{}, fmt.Errorf("can't get absolute path of hub: %+v", err)
}
// we can't use hubtest without the hub
if _, err := os.Stat(hubPath); os.IsNotExist(err) {
if _, err = os.Stat(hubPath); os.IsNotExist(err) {
return HubTest{}, fmt.Errorf("path to hub '%s' doesn't exist, can't run", hubPath)
}
HubTestPath := filepath.Join(hubPath, "./.tests/")
// we can't use hubtest without crowdsec binary
if _, err := exec.LookPath(crowdsecPath); err != nil {
if _, err := os.Stat(crowdsecPath); os.IsNotExist(err) {
if _, err = exec.LookPath(crowdsecPath); err != nil {
if _, err = os.Stat(crowdsecPath); os.IsNotExist(err) {
return HubTest{}, fmt.Errorf("path to crowdsec binary '%s' doesn't exist or is not in $PATH, can't run", crowdsecPath)
}
}
// we can't use hubtest without cscli binary
if _, err := exec.LookPath(cscliPath); err != nil {
if _, err := os.Stat(cscliPath); os.IsNotExist(err) {
if _, err = exec.LookPath(cscliPath); err != nil {
if _, err = os.Stat(cscliPath); os.IsNotExist(err) {
return HubTest{}, fmt.Errorf("path to cscli binary '%s' doesn't exist or is not in $PATH, can't run", cscliPath)
}
}
hubIndexFile := filepath.Join(hubPath, ".index.json")
bidx, err := os.ReadFile(hubIndexFile)
if err != nil {
return HubTest{}, fmt.Errorf("unable to read index file: %s", err)
local := &csconfig.LocalHubCfg{
HubDir: hubPath,
HubIndexFile: hubIndexFile,
InstallDir: HubTestPath,
InstallDataDir: HubTestPath,
}
// load hub index
hubIndex, err := cwhub.LoadPkgIndex(bidx)
hub, err := cwhub.NewHub(local, nil, false)
if err != nil {
return HubTest{}, fmt.Errorf("unable to load hub index file: %s", err)
return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
}
templateConfigFilePath := filepath.Join(HubTestPath, templateConfigFile)
@ -80,16 +83,18 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
TemplateConfigPath: templateConfigFilePath,
TemplateProfilePath: templateProfilePath,
TemplateSimulationPath: templateSimulationPath,
HubIndex: &HubIndex{Data: hubIndex},
HubIndex: hub,
}, nil
}
func (h *HubTest) LoadTestItem(name string) (*HubTestItem, error) {
HubTestItem := &HubTestItem{}
testItem, err := NewTest(name, h)
if err != nil {
return HubTestItem, err
}
h.Tests = append(h.Tests, testItem)
return testItem, nil
@ -108,5 +113,6 @@ func (h *HubTest) LoadAllTests() error {
}
}
}
return nil
}

118
pkg/hubtest/hubtest_item.go
View file

@ -7,11 +7,12 @@ import (
"path/filepath"
"strings"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/parser"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
type HubTestItemConfig struct {
@ -25,10 +26,6 @@ type HubTestItemConfig struct {
OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00
}
type HubIndex struct {
Data map[string]map[string]cwhub.Item
}
type HubTestItem struct {
Name string
Path string
@ -43,7 +40,7 @@ type HubTestItem struct {
RuntimeConfigFilePath string
RuntimeProfileFilePath string
RuntimeSimulationFilePath string
RuntimeHubConfig *csconfig.Hub
RuntimeHubConfig *csconfig.LocalHubCfg
ResultsPath string
ParserResultFile string
@ -56,7 +53,7 @@ type HubTestItem struct {
TemplateConfigPath string
TemplateProfilePath string
TemplateSimulationPath string
HubIndex *HubIndex
HubIndex *cwhub.Hub
Config *HubTestItemConfig
@ -80,8 +77,6 @@ const (
BucketPourResultFileName = "bucketpour-dump.yaml"
)
var crowdsecPatternsFolder = csconfig.DefaultConfigPath("patterns")
func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
testPath := filepath.Join(hubTest.HubTestPath, name)
runtimeFolder := filepath.Join(testPath, "runtime")
@ -91,10 +86,12 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
// read test configuration file
configFileData := &HubTestItemConfig{}
yamlFile, err := os.ReadFile(configFilePath)
if err != nil {
log.Printf("no config file found in '%s': %v", testPath, err)
}
err = yaml.Unmarshal(yamlFile, configFileData)
if err != nil {
return nil, fmt.Errorf("unmarshal: %v", err)
@ -105,6 +102,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
scenarioAssertFilePath := filepath.Join(testPath, ScenarioAssertFileName)
ScenarioAssert := NewScenarioAssert(scenarioAssertFilePath)
return &HubTestItem{
Name: name,
Path: testPath,
@ -121,7 +119,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
ParserResultFile: filepath.Join(resultPath, ParserResultFileName),
ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName),
BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName),
RuntimeHubConfig: &csconfig.Hub{
RuntimeHubConfig: &csconfig.LocalHubCfg{
HubDir: runtimeHubFolder,
HubIndexFile: hubTest.HubIndexFile,
InstallDir: runtimeFolder,
@ -147,23 +145,25 @@ func (t *HubTestItem) InstallHub() error {
if parser == "" {
continue
}
var parserDirDest string
if hubParser, ok := t.HubIndex.Data[cwhub.PARSERS][parser]; ok {
if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
}
parserFileName := filepath.Base(parserSource)
// runtime/hub/parsers/s00-raw/crowdsecurity/
hubDirParserDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubParser.RemotePath))
// runtime/parsers/s00-raw/
parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, hubParser.Stage)
if err := os.MkdirAll(hubDirParserDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirParserDest, err)
}
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
}
@ -204,7 +204,7 @@ func (t *HubTestItem) InstallHub() error {
//return fmt.Errorf("stage '%s' extracted from '%s' doesn't exist in the hub", customParserStage, hubStagePath)
}
parserDirDest = fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
parserDirDest := fmt.Sprintf("%s/parsers/%s/", t.RuntimePath, customParserStage)
if err := os.MkdirAll(parserDirDest, os.ModePerm); err != nil {
continue
//return fmt.Errorf("unable to create folder '%s': %s", parserDirDest, err)
@ -231,23 +231,25 @@ func (t *HubTestItem) InstallHub() error {
if scenario == "" {
continue
}
var scenarioDirDest string
if hubScenario, ok := t.HubIndex.Data[cwhub.SCENARIOS][scenario]; ok {
if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
}
scenarioFileName := filepath.Base(scenarioSource)
// runtime/hub/scenarios/crowdsecurity/
hubDirScenarioDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubScenario.RemotePath))
// runtime/parsers/scenarios/
scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
if err := os.MkdirAll(hubDirScenarioDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirScenarioDest, err)
}
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
}
@ -275,7 +277,7 @@ func (t *HubTestItem) InstallHub() error {
//return fmt.Errorf("scenarios '%s' doesn't exist in the hub and doesn't appear to be a custom one.", scenario)
}
scenarioDirDest = fmt.Sprintf("%s/scenarios/", t.RuntimePath)
scenarioDirDest := fmt.Sprintf("%s/scenarios/", t.RuntimePath)
if err := os.MkdirAll(scenarioDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", scenarioDirDest, err)
}
@ -300,23 +302,25 @@ func (t *HubTestItem) InstallHub() error {
if postoverflow == "" {
continue
}
var postoverflowDirDest string
if hubPostOverflow, ok := t.HubIndex.Data[cwhub.PARSERS_OVFLW][postoverflow]; ok {
if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)
}
postoverflowFileName := filepath.Base(postoverflowSource)
// runtime/hub/postoverflows/s00-enrich/crowdsecurity/
hubDirPostoverflowDest := filepath.Join(t.RuntimeHubPath, filepath.Dir(hubPostOverflow.RemotePath))
// runtime/postoverflows/s00-enrich
postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, hubPostOverflow.Stage)
if err := os.MkdirAll(hubDirPostoverflowDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", hubDirPostoverflowDest, err)
}
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
}
@ -357,7 +361,7 @@ func (t *HubTestItem) InstallHub() error {
//return fmt.Errorf("stage '%s' from extracted '%s' doesn't exist in the hub", customPostoverflowStage, hubStagePath)
}
postoverflowDirDest = fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
postoverflowDirDest := fmt.Sprintf("%s/postoverflows/%s/", t.RuntimePath, customPostoverflowStage)
if err := os.MkdirAll(postoverflowDirDest, os.ModePerm); err != nil {
continue
//return fmt.Errorf("unable to create folder '%s': %s", postoverflowDirDest, err)
@ -384,10 +388,12 @@ func (t *HubTestItem) InstallHub() error {
Filter: "1==1",
Statics: t.Config.OverrideStatics,
}
b, err := yaml.Marshal(n)
if err != nil {
return fmt.Errorf("unable to marshal overrides: %s", err)
}
tgtFilename := fmt.Sprintf("%s/parsers/s00-raw/00_overrides.yaml", t.RuntimePath)
if err := os.WriteFile(tgtFilename, b, os.ModePerm); err != nil {
return fmt.Errorf("unable to write overrides to '%s': %s", tgtFilename, err)
@ -395,40 +401,43 @@ func (t *HubTestItem) InstallHub() error {
}
// load installed hub
err := cwhub.GetHubIdx(t.RuntimeHubConfig)
hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false)
if err != nil {
log.Fatalf("can't local sync the hub: %+v", err)
log.Fatal(err)
}
// install data for parsers if needed
ret := cwhub.GetItemMap(cwhub.PARSERS)
ret := hub.GetItemMap(cwhub.PARSERS)
for parserName, item := range ret {
if item.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
if item.State.Installed {
if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
}
log.Debugf("parser '%s' installed successfully in runtime environment", parserName)
}
}
// install data for scenarios if needed
ret = cwhub.GetItemMap(cwhub.SCENARIOS)
ret = hub.GetItemMap(cwhub.SCENARIOS)
for scenarioName, item := range ret {
if item.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
if item.State.Installed {
if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
}
log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName)
}
}
// install data for postoverflows if needed
ret = cwhub.GetItemMap(cwhub.PARSERS_OVFLW)
ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
for postoverflowName, item := range ret {
if item.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
if item.State.Installed {
if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
}
log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName)
}
}
@ -455,51 +464,53 @@ func (t *HubTestItem) Run() error {
}
// create runtime folder
if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
}
// create runtime data folder
if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
}
// create runtime hub folder
if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
}
if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
}
// create results folder
if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
}
// copy template config file to runtime folder
if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
}
// copy template profile file to runtime folder
if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
}
// copy template simulation file to runtime folder
if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
}
crowdsecPatternsFolder := csconfig.DefaultConfigPath("patterns")
// copy template patterns folder to runtime folder
if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
}
// install the hub in the runtime folder
if err := t.InstallHub(); err != nil {
if err = t.InstallHub(); err != nil {
return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
}
@ -507,7 +518,7 @@ func (t *HubTestItem) Run() error {
logType := t.Config.LogType
dsn := fmt.Sprintf("file://%s", logFile)
if err := os.Chdir(testPath); err != nil {
if err = os.Chdir(testPath); err != nil {
return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
}
@ -515,6 +526,7 @@ func (t *HubTestItem) Run() error {
if err != nil {
return fmt.Errorf("unable to stat log file '%s': %s", logFile, err)
}
if logFileStat.Size() == 0 {
return fmt.Errorf("log file '%s' is empty, please fill it with log", logFile)
}
@ -522,6 +534,7 @@ func (t *HubTestItem) Run() error {
cmdArgs := []string{"-c", t.RuntimeConfigFilePath, "machines", "add", "testMachine", "--auto"}
cscliRegisterCmd := exec.Command(t.CscliPath, cmdArgs...)
log.Debugf("%s", cscliRegisterCmd.String())
output, err := cscliRegisterCmd.CombinedOutput()
if err != nil {
if !strings.Contains(string(output), "unable to create machine: user 'testMachine': user already exist") {
@ -531,16 +544,20 @@ func (t *HubTestItem) Run() error {
}
cmdArgs = []string{"-c", t.RuntimeConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", t.ResultsPath, "-order-event"}
for labelKey, labelValue := range t.Config.Labels {
arg := fmt.Sprintf("%s:%s", labelKey, labelValue)
cmdArgs = append(cmdArgs, "-label", arg)
}
crowdsecCmd := exec.Command(t.CrowdSecPath, cmdArgs...)
log.Debugf("%s", crowdsecCmd.String())
output, err = crowdsecCmd.CombinedOutput()
if log.GetLevel() >= log.DebugLevel || err != nil {
fmt.Println(string(output))
}
if err != nil {
return fmt.Errorf("fail to run '%s' for test '%s': %v", crowdsecCmd.String(), t.Name, err)
}
@ -557,8 +574,10 @@ func (t *HubTestItem) Run() error {
if err != nil {
return err
}
parserAssertFile.Close()
}
assertFileStat, err := os.Stat(t.ParserAssert.File)
if err != nil {
return fmt.Errorf("error while stats '%s': %s", t.ParserAssert.File, err)
@ -569,6 +588,7 @@ func (t *HubTestItem) Run() error {
if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err)
}
t.ParserAssert.AutoGenAssertData = assertData
t.ParserAssert.AutoGenAssert = true
} else {
@ -580,12 +600,15 @@ func (t *HubTestItem) Run() error {
// assert scenarios
nbScenario := 0
for _, scenario := range t.Config.Scenarios {
if scenario == "" {
continue
}
nbScenario += 1
nbScenario++
}
if nbScenario > 0 {
_, err := os.Stat(t.ScenarioAssert.File)
if os.IsNotExist(err) {
@ -593,8 +616,10 @@ func (t *HubTestItem) Run() error {
if err != nil {
return err
}
scenarioAssertFile.Close()
}
assertFileStat, err := os.Stat(t.ScenarioAssert.File)
if err != nil {
return fmt.Errorf("error while stats '%s': %s", t.ScenarioAssert.File, err)
@ -605,6 +630,7 @@ func (t *HubTestItem) Run() error {
if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err)
}
t.ScenarioAssert.AutoGenAssertData = assertData
t.ScenarioAssert.AutoGenAssert = true
} else {

174
pkg/hubtest/parser_assert.go
View file

@ -5,13 +5,11 @@ import (
"fmt"
"io"
"os"
"regexp"
"sort"
"strings"
"time"
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
"github.com/enescakir/emoji"
"github.com/fatih/color"
diff "github.com/r3labs/diff/v2"
@ -43,10 +41,10 @@ type ParserResult struct {
Evt types.Event
Success bool
}
type ParserResults map[string]map[string][]ParserResult
func NewParserAssert(file string) *ParserAssert {
ParserAssert := &ParserAssert{
File: file,
NbAssert: 0,
@ -55,6 +53,7 @@ func NewParserAssert(file string) *ParserAssert {
AutoGenAssert: false,
TestData: &ParserResults{},
}
return ParserAssert
}
@ -63,22 +62,24 @@ func (p *ParserAssert) AutoGenFromFile(filename string) (string, error) {
if err != nil {
return "", err
}
ret := p.AutoGenParserAssert()
return ret, nil
}
func (p *ParserAssert) LoadTest(filename string) error {
var err error
parserDump, err := LoadParserDump(filename)
if err != nil {
return fmt.Errorf("loading parser dump file: %+v", err)
}
p.TestData = parserDump
return nil
}
func (p *ParserAssert) AssertFile(testFile string) error {
file, err := os.Open(p.File)
if err != nil {
@ -88,19 +89,26 @@ func (p *ParserAssert) AssertFile(testFile string) error {
if err := p.LoadTest(testFile); err != nil {
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
}
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
nbLine := 0
for scanner.Scan() {
nbLine += 1
nbLine++
if scanner.Text() == "" {
continue
}
ok, err := p.Run(scanner.Text())
if err != nil {
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
}
p.NbAssert += 1
p.NbAssert++
if !ok {
log.Debugf("%s is FALSE", scanner.Text())
failedAssert := &AssertFail{
@ -109,37 +117,43 @@ func (p *ParserAssert) AssertFile(testFile string) error {
Expression: scanner.Text(),
Debug: make(map[string]string),
}
variableRE := regexp.MustCompile(`(?P<variable>[^ =]+) == .*`)
match := variableRE.FindStringSubmatch(scanner.Text())
variable := ""
if len(match) == 0 {
log.Infof("Couldn't get variable of line '%s'", scanner.Text())
variable = scanner.Text()
} else {
variable = match[1]
}
result, err := p.EvalExpression(variable)
if err != nil {
log.Errorf("unable to evaluate variable '%s': %s", variable, err)
continue
}
failedAssert.Debug[variable] = result
p.Fails = append(p.Fails, *failedAssert)
continue
}
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
}
file.Close()
if p.NbAssert == 0 {
assertData, err := p.AutoGenFromFile(testFile)
if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err)
}
p.AutoGenAssertData = assertData
p.AutoGenAssert = true
}
if len(p.Fails) == 0 {
p.Success = true
}
@ -148,15 +162,14 @@ func (p *ParserAssert) AssertFile(testFile string) error {
}
func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
var err error
//debug doesn't make much sense with the ability to evaluate "on the fly"
//var debugFilter *exprhelpers.ExprDebugger
var runtimeFilter *vm.Program
var output interface{}
env := map[string]interface{}{"results": *p.TestData}
if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
if err != nil {
log.Errorf("failed to compile '%s' : %s", expression, err)
return output, err
}
@ -168,8 +181,10 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
if err != nil {
log.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err)
return output, fmt.Errorf("while running expression %s: %w", expression, err)
}
return output, nil
}
@ -178,10 +193,13 @@ func (p *ParserAssert) EvalExpression(expression string) (string, error) {
if err != nil {
return "", err
}
ret, err := yaml.Marshal(output)
if err != nil {
return "", err
}
return string(ret), nil
}
@ -190,6 +208,7 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
if err != nil {
return false, err
}
switch out := output.(type) {
case bool:
return out, nil
@ -201,80 +220,89 @@ func (p *ParserAssert) Run(assert string) (bool, error) {
func Escape(val string) string {
val = strings.ReplaceAll(val, `\`, `\\`)
val = strings.ReplaceAll(val, `"`, `\"`)
return val
}
func (p *ParserAssert) AutoGenParserAssert() string {
//attempt to autogen parser asserts
var ret string
ret := fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
//sort map keys for consistent order
stages := sortedMapKeys(*p.TestData)
//sort map keys for consistent ordre
var stages []string
for stage := range *p.TestData {
stages = append(stages, stage)
}
sort.Strings(stages)
ret += fmt.Sprintf("len(results) == %d\n", len(*p.TestData))
for _, stage := range stages {
parsers := (*p.TestData)[stage]
//sort map keys for consistent ordre
var pnames []string
for pname := range parsers {
pnames = append(pnames, pname)
}
sort.Strings(pnames)
//sort map keys for consistent order
pnames := sortedMapKeys(parsers)
for _, parser := range pnames {
presults := parsers[parser]
ret += fmt.Sprintf(`len(results["%s"]["%s"]) == %d`+"\n", stage, parser, len(presults))
for pidx, result := range presults {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Success == %t`+"\n", stage, parser, pidx, result.Success)
if !result.Success {
continue
}
for _, pkey := range sortedMapKeys(result.Evt.Parsed) {
pval := result.Evt.Parsed[pkey]
if pval == "" {
continue
}
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Parsed["%s"] == "%s"`+"\n", stage, parser, pidx, pkey, Escape(pval))
}
for _, mkey := range sortedMapKeys(result.Evt.Meta) {
mval := result.Evt.Meta[mkey]
if mval == "" {
continue
}
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Meta["%s"] == "%s"`+"\n", stage, parser, pidx, mkey, Escape(mval))
}
for _, ekey := range sortedMapKeys(result.Evt.Enriched) {
eval := result.Evt.Enriched[ekey]
if eval == "" {
continue
}
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Enriched["%s"] == "%s"`+"\n", stage, parser, pidx, ekey, Escape(eval))
}
for _, ukey := range sortedMapKeys(result.Evt.Unmarshaled) {
uval := result.Evt.Unmarshaled[ukey]
if uval == "" {
continue
}
base := fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Unmarshaled["%s"]`, stage, parser, pidx, ukey)
for _, line := range p.buildUnmarshaledAssert(base, uval) {
ret += line
}
}
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.Whitelisted == %t`+"\n", stage, parser, pidx, result.Evt.Whitelisted)
if result.Evt.WhitelistReason != "" {
ret += fmt.Sprintf(`results["%s"]["%s"][%d].Evt.WhitelistReason == "%s"`+"\n", stage, parser, pidx, Escape(result.Evt.WhitelistReason))
}
}
}
}
return ret
}
func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []string {
ret := make([]string, 0)
switch val := eval.(type) {
case map[string]interface{}:
for k, v := range val {
@ -297,12 +325,11 @@ func (p *ParserAssert) buildUnmarshaledAssert(ekey string, eval interface{}) []s
default:
log.Warningf("unknown type '%T' for key '%s'", val, ekey)
}
return ret
}
func LoadParserDump(filepath string) (*ParserResults, error) {
var pdump ParserResults
dumpData, err := os.Open(filepath)
if err != nil {
return nil, err
@ -314,18 +341,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
return nil, err
}
pdump := ParserResults{}
if err := yaml.Unmarshal(results, &pdump); err != nil {
return nil, err
}
/* we know that some variables should always be set,
let's check if they're present in last parser output of last stage */
stages := make([]string, 0, len(pdump))
for k := range pdump {
stages = append(stages, k)
}
sort.Strings(stages)
stages := sortedMapKeys(pdump)
var lastStage string
//Loop over stages to find last successful one with at least one parser
for i := len(stages) - 2; i >= 0; i-- {
if len(pdump[stages[i]]) != 0 {
@ -333,11 +361,19 @@ func LoadParserDump(filepath string) (*ParserResults, error) {
break
}
}
parsers := make([]string, 0, len(pdump[lastStage]))
for k := range pdump[lastStage] {
parsers = append(parsers, k)
}
sort.Strings(parsers)
if len(parsers) == 0 {
return nil, fmt.Errorf("no parser found. Please install the appropriate parser and retry")
}
lastParser := parsers[len(parsers)-1]
for idx, result := range pdump[lastStage][lastParser] {
@ -357,47 +393,51 @@ type DumpOpts struct {
ShowNotOkParsers bool
}
func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts DumpOpts) {
func DumpTree(parserResults ParserResults, bucketPour BucketPourInfo, opts DumpOpts) {
//note : we can use line -> time as the unique identifier (of acquisition)
state := make(map[time.Time]map[string]map[string]ParserResult)
assoc := make(map[time.Time]string, 0)
for stage, parsers := range parser_results {
for stage, parsers := range parserResults {
for parser, results := range parsers {
for _, parser_res := range results {
evt := parser_res.Evt
for _, parserRes := range results {
evt := parserRes.Evt
if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw
}
if _, ok := state[evt.Line.Time][stage]; !ok {
state[evt.Line.Time][stage] = make(map[string]ParserResult)
}
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parser_res.Success}
}
state[evt.Line.Time][stage][parser] = ParserResult{Evt: evt, Success: parserRes.Success}
}
}
}
for bname, evtlist := range bucket_pour {
for bname, evtlist := range bucketPour {
for _, evt := range evtlist {
if evt.Line.Raw == "" {
continue
}
//it might be bucket overflow being reprocessed, skip this
if _, ok := state[evt.Line.Time]; !ok {
state[evt.Line.Time] = make(map[string]map[string]ParserResult)
assoc[evt.Line.Time] = evt.Line.Raw
}
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if _, ok := state[evt.Line.Time]["buckets"]; !ok {
state[evt.Line.Time]["buckets"] = make(map[string]ParserResult)
}
state[evt.Line.Time]["buckets"][bname] = ParserResult{Success: true}
}
}
yellow := color.New(color.FgYellow).SprintFunc()
red := color.New(color.FgRed).SprintFunc()
green := color.New(color.FgGreen).SprintFunc()
@ -409,19 +449,25 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
continue
}
}
fmt.Printf("line: %s\n", rawstr)
skeys := make([]string, 0, len(state[tstamp]))
for k := range state[tstamp] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "buckets" {
continue
}
skeys = append(skeys, k)
}
sort.Strings(skeys)
//iterate stage
var prev_item types.Event
// iterate stage
var prevItem types.Event
for _, stage := range skeys {
parsers := state[tstamp][stage]
@ -431,18 +477,16 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
fmt.Printf("\t%s %s\n", sep, stage)
pkeys := make([]string, 0, len(parsers))
for k := range parsers {
pkeys = append(pkeys, k)
}
sort.Strings(pkeys)
pkeys := sortedMapKeys(parsers)
for idx, parser := range pkeys {
res := parsers[parser].Success
sep := "├"
if idx == len(pkeys)-1 {
sep = "└"
}
created := 0
updated := 0
deleted := 0
@ -451,16 +495,19 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
detailsDisplay := ""
if res {
changelog, _ := diff.Diff(prev_item, parsers[parser].Evt)
changelog, _ := diff.Diff(prevItem, parsers[parser].Evt)
for _, change := range changelog {
switch change.Type {
case "create":
created++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), green(change.To))
case "update":
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s : %s -> %s\n", presep, sep, change.Type, strings.Join(change.Path, "."), change.From, yellow(change.To))
if change.Path[0] == "Whitelisted" && change.To == true {
whitelisted = true
if whitelistReason == "" {
whitelistReason = parsers[parser].Evt.WhitelistReason
}
@ -468,51 +515,64 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
updated++
case "delete":
deleted++
detailsDisplay += fmt.Sprintf("\t%s\t\t%s %s evt.%s\n", presep, sep, change.Type, red(strings.Join(change.Path, ".")))
}
}
prev_item = parsers[parser].Evt
prevItem = parsers[parser].Evt
}
if created > 0 {
changeStr += green(fmt.Sprintf("+%d", created))
}
if updated > 0 {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += yellow(fmt.Sprintf("~%d", updated))
}
if deleted > 0 {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += red(fmt.Sprintf("-%d", deleted))
}
if whitelisted {
if len(changeStr) > 0 {
changeStr += " "
}
changeStr += red("[whitelisted]")
}
if changeStr == "" {
changeStr = yellow("unchanged")
}
if res {
fmt.Printf("\t%s\t%s %s %s (%s)\n", presep, sep, emoji.GreenCircle, parser, changeStr)
if opts.Details {
fmt.Print(detailsDisplay)
}
} else if opts.ShowNotOkParsers {
fmt.Printf("\t%s\t%s %s %s\n", presep, sep, emoji.RedCircle, parser)
}
}
}
}
}
}
sep := "└"
if len(state[tstamp]["buckets"]) > 0 {
sep = "├"
}
//did the event enter the bucket pour phase ?
if _, ok := state[tstamp]["buckets"]["OK"]; ok {
fmt.Printf("\t%s-------- parser success %s\n", sep, emoji.GreenCircle)
@ -521,27 +581,35 @@ func DumpTree(parser_results ParserResults, bucket_pour BucketPourInfo, opts Dum
} else {
fmt.Printf("\t%s-------- parser failure %s\n", sep, emoji.RedCircle)
}
//now print bucket info
if len(state[tstamp]["buckets"]) > 0 {
fmt.Printf("\t├ Scenarios\n")
}
bnames := make([]string, 0, len(state[tstamp]["buckets"]))
for k := range state[tstamp]["buckets"] {
//there is a trick : to know if an event successfully exit the parsers, we check if it reached the pour() phase
//we thus use a fake stage "buckets" and a fake parser "OK" to know if it entered
if k == "OK" {
continue
}
bnames = append(bnames, k)
}
sort.Strings(bnames)
for idx, bname := range bnames {
sep := "├"
if idx == len(bnames)-1 {
sep = "└"
}
fmt.Printf("\t\t%s %s %s\n", sep, emoji.GreenCircle, bname)
}
fmt.Println()
}
}

11
pkg/hubtest/regexp.go Normal file
View file

@ -0,0 +1,11 @@
package hubtest
import (
"regexp"
)
var (
variableRE = regexp.MustCompile(`(?P<variable>[^ =]+) == .*`)
parserResultRE = regexp.MustCompile(`^results\["[^"]+"\]\["(?P<parser>[^"]+)"\]\[[0-9]+\]\.Evt\..*`)
scenarioResultRE = regexp.MustCompile(`^results\[[0-9]+\].Overflow.Alert.GetScenario\(\) == "(?P<scenario>[^"]+)"`)
)

57
pkg/hubtest/scenario_assert.go
View file

@ -5,12 +5,10 @@ import (
"fmt"
"io"
"os"
"regexp"
"sort"
"strings"
"github.com/antonmedv/expr"
"github.com/antonmedv/expr/vm"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
@ -42,6 +40,7 @@ func NewScenarioAssert(file string) *ScenarioAssert {
TestData: &BucketResults{},
PourData: &BucketPourInfo{},
}
return ScenarioAssert
}
@ -50,7 +49,9 @@ func (s *ScenarioAssert) AutoGenFromFile(filename string) (string, error) {
if err != nil {
return "", err
}
ret := s.AutoGenScenarioAssert()
return ret, nil
}
@ -59,6 +60,7 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
if err != nil {
return fmt.Errorf("loading scenario dump file '%s': %+v", filename, err)
}
s.TestData = bucketDump
if bucketpour != "" {
@ -66,8 +68,10 @@ func (s *ScenarioAssert) LoadTest(filename string, bucketpour string) error {
if err != nil {
return fmt.Errorf("loading bucket pour dump file '%s': %+v", filename, err)
}
s.PourData = pourDump
}
return nil
}
@ -81,19 +85,26 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
if err := s.LoadTest(testFile, ""); err != nil {
return fmt.Errorf("unable to load parser dump file '%s': %s", testFile, err)
}
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
nbLine := 0
for scanner.Scan() {
nbLine += 1
nbLine++
if scanner.Text() == "" {
continue
}
ok, err := s.Run(scanner.Text())
if err != nil {
return fmt.Errorf("unable to run assert '%s': %+v", scanner.Text(), err)
}
s.NbAssert += 1
s.NbAssert++
if !ok {
log.Debugf("%s is FALSE", scanner.Text())
failedAssert := &AssertFail{
@ -102,31 +113,38 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
Expression: scanner.Text(),
Debug: make(map[string]string),
}
variableRE := regexp.MustCompile(`(?P<variable>[^ ]+) == .*`)
match := variableRE.FindStringSubmatch(scanner.Text())
if len(match) == 0 {
log.Infof("Couldn't get variable of line '%s'", scanner.Text())
continue
}
variable := match[1]
result, err := s.EvalExpression(variable)
if err != nil {
log.Errorf("unable to evaluate variable '%s': %s", variable, err)
continue
}
failedAssert.Debug[variable] = result
s.Fails = append(s.Fails, *failedAssert)
continue
}
//fmt.Printf(" %s '%s'\n", emoji.GreenSquare, scanner.Text())
}
file.Close()
if s.NbAssert == 0 {
assertData, err := s.AutoGenFromFile(testFile)
if err != nil {
return fmt.Errorf("couldn't generate assertion: %s", err)
}
s.AutoGenAssertData = assertData
s.AutoGenAssert = true
}
@ -139,15 +157,14 @@ func (s *ScenarioAssert) AssertFile(testFile string) error {
}
func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
var err error
//debug doesn't make much sense with the ability to evaluate "on the fly"
//var debugFilter *exprhelpers.ExprDebugger
var runtimeFilter *vm.Program
var output interface{}
env := map[string]interface{}{"results": *s.TestData}
if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
runtimeFilter, err := expr.Compile(expression, exprhelpers.GetExprOptions(env)...)
if err != nil {
return nil, err
}
// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil {
@ -161,8 +178,10 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
if err != nil {
log.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err)
return nil, fmt.Errorf("while running expression %s: %w", expression, err)
}
return output, nil
}
@ -171,10 +190,12 @@ func (s *ScenarioAssert) EvalExpression(expression string) (string, error) {
if err != nil {
return "", err
}
ret, err := yaml.Marshal(output)
if err != nil {
return "", err
}
return string(ret), nil
}
@ -183,6 +204,7 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
if err != nil {
return false, err
}
switch out := output.(type) {
case bool:
return out, nil
@ -192,9 +214,9 @@ func (s *ScenarioAssert) Run(assert string) (bool, error) {
}
func (s *ScenarioAssert) AutoGenScenarioAssert() string {
//attempt to autogen parser asserts
var ret string
ret += fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
// attempt to autogen scenario asserts
ret := fmt.Sprintf(`len(results) == %d`+"\n", len(*s.TestData))
for eventIndex, event := range *s.TestData {
for ipSrc, source := range event.Overflow.Sources {
ret += fmt.Sprintf(`"%s" in results[%d].Overflow.GetSources()`+"\n", ipSrc, eventIndex)
@ -203,15 +225,18 @@ func (s *ScenarioAssert) AutoGenScenarioAssert() string {
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetScope() == "%s"`+"\n", eventIndex, ipSrc, *source.Scope)
ret += fmt.Sprintf(`results[%d].Overflow.Sources["%s"].GetValue() == "%s"`+"\n", eventIndex, ipSrc, *source.Value)
}
for evtIndex, evt := range event.Overflow.Alert.Events {
for _, meta := range evt.Meta {
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Events[%d].GetMeta("%s") == "%s"`+"\n", eventIndex, evtIndex, meta.Key, Escape(meta.Value))
}
}
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetScenario() == "%s"`+"\n", eventIndex, *event.Overflow.Alert.Scenario)
ret += fmt.Sprintf(`results[%d].Overflow.Alert.Remediation == %t`+"\n", eventIndex, event.Overflow.Alert.Remediation)
ret += fmt.Sprintf(`results[%d].Overflow.Alert.GetEventsCount() == %d`+"\n", eventIndex, *event.Overflow.Alert.EventsCount)
}
return ret
}
@ -228,8 +253,6 @@ func (b BucketResults) Swap(i, j int) {
}
func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
var bucketDump BucketPourInfo
dumpData, err := os.Open(filepath)
if err != nil {
return nil, err
@ -241,6 +264,8 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
return nil, err
}
var bucketDump BucketPourInfo
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
return nil, err
}
@ -249,8 +274,6 @@ func LoadBucketPourDump(filepath string) (*BucketPourInfo, error) {
}
func LoadScenarioDump(filepath string) (*BucketResults, error) {
var bucketDump BucketResults
dumpData, err := os.Open(filepath)
if err != nil {
return nil, err
@ -262,6 +285,8 @@ func LoadScenarioDump(filepath string) (*BucketResults, error) {
return nil, err
}
var bucketDump BucketResults
if err := yaml.Unmarshal(results, &bucketDump); err != nil {
return nil, err
}

8
pkg/hubtest/utils.go
View file

@ -12,7 +12,9 @@ func sortedMapKeys[V any](m map[string]V) []string {
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
@ -22,7 +24,7 @@ func Copy(src string, dst string) error {
return err
}
err = os.WriteFile(dst, content, 0644)
err = os.WriteFile(dst, content, 0o644)
if err != nil {
return err
}
@ -43,16 +45,20 @@ func checkPathNotContained(path string, subpath string) error {
}
current := absSubPath
for {
if current == absPath {
return fmt.Errorf("cannot copy a folder onto itself")
}
up := filepath.Dir(current)
if current == up {
break
}
current = up
}
return nil
}

18
pkg/hubtest/utils_test.go
View file

@ -3,16 +3,16 @@ package hubtest
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCheckPathNotContained(t *testing.T) {
assert.Nil(t, checkPathNotContained("/foo", "/bar"))
assert.Nil(t, checkPathNotContained("/foo/bar", "/foo"))
assert.Nil(t, checkPathNotContained("/foo/bar", "/"))
assert.Nil(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else"))
assert.Nil(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else"))
assert.NotNil(t, checkPathNotContained("/foo", "/foo/bar"))
assert.NotNil(t, checkPathNotContained("/", "/foo"))
assert.NotNil(t, checkPathNotContained("/", "/foo/bar/baz"))
require.NoError(t, checkPathNotContained("/foo", "/bar"))
require.NoError(t, checkPathNotContained("/foo/bar", "/foo"))
require.NoError(t, checkPathNotContained("/foo/bar", "/"))
require.NoError(t, checkPathNotContained("/path/to/somewhere", "/path/to/somewhere-else"))
require.NoError(t, checkPathNotContained("~/.local/path/to/somewhere", "~/.local/path/to/somewhere-else"))
require.Error(t, checkPathNotContained("/foo", "/foo/bar"))
require.Error(t, checkPathNotContained("/", "/foo"))
require.Error(t, checkPathNotContained("/", "/foo/bar/baz"))
}

39
pkg/leakybucket/buckets_test.go
View file

@ -8,12 +8,14 @@ import (
"html/template"
"io"
"os"
"path/filepath"
"reflect"
"sync"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -33,28 +35,45 @@ func TestBucket(t *testing.T) {
envSetting = os.Getenv("TEST_ONLY")
tomb = &tomb.Tomb{}
)
err := exprhelpers.Init(nil)
testdata := "./tests"
hubCfg := &csconfig.LocalHubCfg{
HubDir: filepath.Join(testdata, "hub"),
HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
InstallDataDir: testdata,
}
hub, err := cwhub.NewHub(hubCfg, nil, false)
if err != nil {
t.Fatalf("failed to init hub: %s", err)
}
err = exprhelpers.Init(nil)
if err != nil {
log.Fatalf("exprhelpers init failed: %s", err)
}
if envSetting != "" {
if err := testOneBucket(t, envSetting, tomb); err != nil {
if err := testOneBucket(t, hub, envSetting, tomb); err != nil {
t.Fatalf("Test '%s' failed : %s", envSetting, err)
}
} else {
wg := new(sync.WaitGroup)
fds, err := os.ReadDir("./tests/")
fds, err := os.ReadDir(testdata)
if err != nil {
t.Fatalf("Unable to read test directory : %s", err)
}
for _, fd := range fds {
fname := "./tests/" + fd.Name()
if fd.Name() == "hub" {
continue
}
fname := filepath.Join(testdata, fd.Name())
log.Infof("Running test on %s", fname)
tomb.Go(func() error {
wg.Add(1)
defer wg.Done()
if err := testOneBucket(t, fname, tomb); err != nil {
if err := testOneBucket(t, hub, fname, tomb); err != nil {
t.Fatalf("Test '%s' failed : %s", fname, err)
}
return nil
@ -76,7 +95,7 @@ func watchTomb(tomb *tomb.Tomb) {
}
}
func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) error {
var (
holders []BucketFactory
@ -112,10 +131,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
files = append(files, x.Filename)
}
cscfg := &csconfig.CrowdsecServiceCfg{
DataDir: "tests",
}
holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false)
cscfg := &csconfig.CrowdsecServiceCfg{}
holders, response, err := LoadBuckets(cscfg, hub, files, tomb, buckets, false)
if err != nil {
t.Fatalf("failed loading bucket : %s", err)
}
@ -123,7 +140,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
watchTomb(tomb)
return nil
})
if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) {
if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) {
return fmt.Errorf("tests from %s failed", dir)
}
return nil

10
pkg/leakybucket/manager_load.go
View file

@ -178,7 +178,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
return nil
}
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
var (
ret = []BucketFactory{}
response chan types.Event
@ -211,7 +211,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
log.Tracef("End of yaml file")
break
}
bucketFactory.DataDir = cscfg.DataDir
bucketFactory.DataDir = hub.GetDataDir()
//check empty
if bucketFactory.Name == "" {
log.Errorf("Won't load nameless bucket")
@ -234,7 +234,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
bucketFactory.Filename = filepath.Clean(f)
bucketFactory.BucketName = seed.Generate()
bucketFactory.ret = response
hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
if err != nil {
log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
} else {
@ -242,8 +242,8 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
bucketFactory.Simulated = cscfg.SimulationConfig.IsSimulated(hubItem.Name)
}
if hubItem != nil {
bucketFactory.ScenarioVersion = hubItem.LocalVersion
bucketFactory.hash = hubItem.LocalHash
bucketFactory.ScenarioVersion = hubItem.State.LocalVersion
bucketFactory.hash = hubItem.State.LocalHash
} else {
log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
}

1
pkg/leakybucket/tests/hub/index.json Normal file
View file

@ -0,0 +1 @@
{}

21
pkg/parser/unix_parser.go
View file

@ -57,24 +57,25 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
// Return new parsers
// nodes and povfwnodes are already initialized in parser.LoadStages
func NewParsers() *Parsers {
func NewParsers(hub *cwhub.Hub) *Parsers {
parsers := &Parsers{
Ctx: &UnixParserCtx{},
Povfwctx: &UnixParserCtx{},
StageFiles: make([]Stagefile, 0),
PovfwStageFiles: make([]Stagefile, 0),
}
for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
for _, hubParserItem := range cwhub.GetItemMap(itemType) {
if hubParserItem.Installed {
for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
for _, hubParserItem := range hub.GetItemMap(itemType) {
if hubParserItem.State.Installed {
stagefile := Stagefile{
Filename: hubParserItem.LocalPath,
Filename: hubParserItem.State.LocalPath,
Stage: hubParserItem.Stage,
}
if itemType == cwhub.PARSERS {
parsers.StageFiles = append(parsers.StageFiles, stagefile)
}
if itemType == cwhub.PARSERS_OVFLW {
if itemType == cwhub.POSTOVERFLOWS {
parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
}
}
@ -97,16 +98,16 @@ func NewParsers() *Parsers {
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error
patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/")
log.Infof("Loading grok library %s", patternsDir)
/* load base regexps for two grok parsers */
parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.Crowdsec.DataDir})
"data": cConfig.ConfigPaths.DataDir})
if err != nil {
return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
}
parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.Crowdsec.DataDir})
"data": cConfig.ConfigPaths.DataDir})
if err != nil {
return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
}
@ -116,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
*/
log.Infof("Loading enrich plugins")
parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
if err != nil {
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
}

49
pkg/setup/install.go
View file

@ -10,7 +10,6 @@ import (
goccyyaml "github.com/goccy/go-yaml"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -46,22 +45,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
}
// InstallHubItems installs the objects recommended in a setup file.
func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error {
func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
setupEnvelope, err := decodeSetup(input, false)
if err != nil {
return err
}
if err := csConfig.LoadHub(); err != nil {
return fmt.Errorf("loading hub: %w", err)
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("getting hub index: %w", err)
}
for _, setupItem := range setupEnvelope.Setup {
forceAction := false
downloadOnly := false
@ -73,14 +62,19 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
if len(install.Collections) > 0 {
for _, collection := range setupItem.Install.Collections {
item := hub.GetItem(cwhub.COLLECTIONS, collection)
if item == nil {
return fmt.Errorf("collection %s not found", collection)
}
if dryRun {
fmt.Println("dry-run: would install collection", collection)
continue
}
if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", collection, err)
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", item.Name, err)
}
}
}
@ -93,8 +87,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", parser, err)
item := hub.GetItem(cwhub.PARSERS, parser)
if item == nil {
return fmt.Errorf("parser %s not found", parser)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", item.Name, err)
}
}
}
@ -107,8 +106,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", scenario, err)
item := hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
return fmt.Errorf("scenario %s not found", scenario)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", item.Name, err)
}
}
}
@ -121,8 +125,13 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow)
if item == nil {
return fmt.Errorf("postoverflow %s not found", postoverflow)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err)
}
}
}

71
test/bats/00_wait_for.bats Normal file
View file

@ -0,0 +1,71 @@
#!/usr/bin/env bats
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
set -u
setup_file() {
load "../lib/setup_file.sh"
}
setup() {
load "../lib/setup.sh"
}
@test "run a command and capture its stdout" {
run -0 wait-for seq 1 3
assert_output - <<-EOT
1
2
3
EOT
}
@test "run a command and capture its stderr" {
rune -0 wait-for sh -c 'seq 1 3 >&2'
assert_stderr - <<-EOT
1
2
3
EOT
}
@test "run a command until a pattern is found in stdout" {
run -0 wait-for --out "1[12]0" seq 1 200
assert_line --index 0 "1"
assert_line --index -1 "110"
refute_line "111"
}
@test "run a command until a pattern is found in stderr" {
rune -0 wait-for --err "10" sh -c 'seq 1 20 >&2'
assert_stderr - <<-EOT
1
2
3
4
5
6
7
8
9
10
EOT
}
@test "run a command with timeout (no match)" {
# when the process is terminated without a match, it returns
# 256 - 15 (SIGTERM) = 241
rune -241 wait-for --timeout 0.1 --out "10" sh -c 'echo 1; sleep 3; echo 2'
assert_line 1
# there may be more, but we don't care
}
@test "run a command with timeout (match)" {
# when the process is terminated with a match, return code is 128
rune -128 wait-for --timeout .4 --out "2" sh -c 'echo 1; sleep .1; echo 2; echo 3; echo 4; sleep 10'
assert_output - <<-EOT
1
2
EOT
}

57
test/bats/01_crowdsec.bats
View file

@ -24,28 +24,22 @@ teardown() {
#----------
@test "crowdsec (usage)" {
rune -0 timeout 2s "${CROWDSEC}" -h
assert_stderr_line --regexp "Usage of .*:"
rune -0 timeout 2s "${CROWDSEC}" --help
assert_stderr_line --regexp "Usage of .*:"
rune -0 wait-for --out "Usage of " "${CROWDSEC}" -h
rune -0 wait-for --out "Usage of " "${CROWDSEC}" --help
}
@test "crowdsec (unknown flag)" {
rune -2 timeout 2s "${CROWDSEC}" --foobar
assert_stderr_line "flag provided but not defined: -foobar"
assert_stderr_line --regexp "Usage of .*"
rune -0 wait-for --err "flag provided but not defined: -foobar" "$CROWDSEC" --foobar
}
@test "crowdsec (unknown argument)" {
rune -2 timeout 2s "${CROWDSEC}" trololo
assert_stderr_line "argument provided but not defined: trololo"
assert_stderr_line --regexp "Usage of .*"
rune -0 wait-for --err "argument provided but not defined: trololo" "${CROWDSEC}" trololo
}
@test "crowdsec (no api and no agent)" {
rune -1 timeout 2s "${CROWDSEC}" -no-api -no-cs
assert_stderr_line --partial "You must run at least the API Server or crowdsec"
rune -0 wait-for \
--err "You must run at least the API Server or crowdsec" \
"${CROWDSEC}" -no-api -no-cs
}
@test "crowdsec - print error on exit" {
@ -55,20 +49,22 @@ teardown() {
assert_stderr --partial "unable to create database client: unknown database type 'meh'"
}
@test "crowdsec - bad configuration (empty/missing common section)" {
@test "crowdsec - default logging configuration (empty/missing common section)" {
config_set '.common={}'
rune -1 "${CROWDSEC}"
rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
refute_output
assert_stderr --partial "unable to load configuration: common section is empty"
config_set 'del(.common)'
rune -1 "${CROWDSEC}"
rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
refute_output
assert_stderr --partial "unable to load configuration: common section is empty"
}
@test "CS_LAPI_SECRET not strong enough" {
CS_LAPI_SECRET=foo rune -1 timeout 2s "${CROWDSEC}"
CS_LAPI_SECRET=foo rune -1 wait-for "${CROWDSEC}"
assert_stderr --partial "api server init: unable to run local API: controller init: CS_LAPI_SECRET not strong enough"
}
@ -138,8 +134,8 @@ teardown() {
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
rm -f "$ACQUIS_YAML"
rune -1 timeout 2s "${CROWDSEC}"
assert_stderr_line --partial "acquis.yaml: no such file or directory"
rune -1 wait-for "${CROWDSEC}"
assert_stderr --partial "acquis.yaml: no such file or directory"
}
@test "crowdsec (error if acquisition_path is not defined and acquisition_dir is empty)" {
@ -151,7 +147,7 @@ teardown() {
rm -f "$ACQUIS_DIR"
config_set '.common.log_media="stdout"'
rune -1 timeout 2s "${CROWDSEC}"
rune -1 wait-for "${CROWDSEC}"
# check warning
assert_stderr --partial "no acquisition file found"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
@ -167,13 +163,15 @@ teardown() {
config_set '.crowdsec_service.acquisition_dir=""'
config_set '.common.log_media="stdout"'
rune -1 timeout 2s "${CROWDSEC}"
rune -1 wait-for "${CROWDSEC}"
# check warning
assert_stderr --partial "no acquisition_path or acquisition_dir specified"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
}
@test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
config_set '.common.log_media="stdout"'
ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
config_set '.crowdsec_service.acquisition_path=""'
@ -181,13 +179,15 @@ teardown() {
mkdir -p "$ACQUIS_DIR"
mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
rune -124 timeout 2s "${CROWDSEC}"
rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
# now, if foo.yaml is empty instead, there won't be valid datasources.
cat /dev/null >"$ACQUIS_DIR"/foo.yaml
rune -1 timeout 2s "${CROWDSEC}"
rune -1 wait-for "${CROWDSEC}"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
}
@ -212,9 +212,10 @@ teardown() {
type: syslog
EOT
rune -124 timeout 2s env PATH='' "${CROWDSEC}"
#shellcheck disable=SC2016
assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
rune -0 wait-for \
--err 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in ' \
env PATH='' "${CROWDSEC}"
# if all datasources are disabled, crowdsec should exit
@ -222,7 +223,7 @@ teardown() {
rm -f "$ACQUIS_YAML"
config_set '.crowdsec_service.acquisition_path=""'
rune -1 timeout 2s env PATH='' "${CROWDSEC}"
rune -1 wait-for env PATH='' "${CROWDSEC}"
assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
}

78
test/bats/01_cscli.bats
View file

@ -110,6 +110,37 @@ teardown() {
assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]'
}
@test "cscli - required configuration paths" {
config=$(cat "${CONFIG_YAML}")
configdir=$(config_get '.config_paths.config_dir')
# required configuration paths with no defaults
config_set 'del(.config_paths)'
rune -1 cscli hub list
assert_stderr --partial 'no configuration paths provided'
echo "$config" > "${CONFIG_YAML}"
config_set 'del(.config_paths.data_dir)'
rune -1 cscli hub list
assert_stderr --partial "please provide a data directory with the 'data_dir' directive in the 'config_paths' section"
echo "$config" > "${CONFIG_YAML}"
# defaults
config_set 'del(.config_paths.hub_dir)'
rune -0 cscli hub list
rune -0 cscli config show --key Config.ConfigPaths.HubDir
assert_output "$configdir/hub"
echo "$config" > "${CONFIG_YAML}"
config_set 'del(.config_paths.index_path)'
rune -0 cscli hub list
rune -0 cscli config show --key Config.ConfigPaths.HubIndexFile
assert_output "$configdir/hub/.index.json"
echo "$config" > "${CONFIG_YAML}"
}
@test "cscli config show-yaml" {
rune -0 cscli config show-yaml
rune -0 yq .common.log_level <(output)
@ -245,50 +276,23 @@ teardown() {
assert_output --partial "# bash completion for cscli"
}
@test "cscli hub list" {
# we check for the presence of some objects. There may be others when we
# use $PACKAGE_TESTING, so the order is not important.
rune -0 cscli hub list -o human
assert_line --regexp '^ crowdsecurity/linux'
assert_line --regexp '^ crowdsecurity/sshd'
assert_line --regexp '^ crowdsecurity/dateparse-enrich'
assert_line --regexp '^ crowdsecurity/geoip-enrich'
assert_line --regexp '^ crowdsecurity/sshd-logs'
assert_line --regexp '^ crowdsecurity/syslog-logs'
assert_line --regexp '^ crowdsecurity/ssh-bf'
assert_line --regexp '^ crowdsecurity/ssh-slow-bf'
rune -0 cscli hub list -o raw
assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$'
assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$'
assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$'
assert_line --regexp '^crowdsecurity/geoip-enrich,enabled,[0-9]+\.[0-9]+,"Populate event with geoloc info : as, country, coords, source range.",parsers$'
assert_line --regexp '^crowdsecurity/sshd-logs,enabled,[0-9]+\.[0-9]+,Parse openSSH logs,parsers$'
assert_line --regexp '^crowdsecurity/syslog-logs,enabled,[0-9]+\.[0-9]+,,parsers$'
assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$'
assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$'
rune -0 cscli hub list -o json
rune -0 jq -r '.collections[].name, .parsers[].name, .scenarios[].name' <(output)
assert_line 'crowdsecurity/linux'
assert_line 'crowdsecurity/sshd'
assert_line 'crowdsecurity/dateparse-enrich'
assert_line 'crowdsecurity/geoip-enrich'
assert_line 'crowdsecurity/sshd-logs'
assert_line 'crowdsecurity/syslog-logs'
assert_line 'crowdsecurity/ssh-bf'
assert_line 'crowdsecurity/ssh-slow-bf'
}
@test "cscli support dump (smoke test)" {
rune -0 cscli support dump -f "$BATS_TEST_TMPDIR"/dump.zip
assert_file_exists "$BATS_TEST_TMPDIR"/dump.zip
}
@test "cscli explain" {
rune -0 cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog --crowdsec "$CROWDSEC"
line="Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4"
rune -0 cscli parsers install crowdsecurity/syslog-logs
rune -0 cscli collections install crowdsecurity/sshd
rune -0 cscli explain --log "$line" --type syslog --only-successful-parsers --crowdsec "$CROWDSEC"
assert_output - <"$BATS_TEST_DIRNAME"/testdata/explain/explain-log.txt
rune -0 cscli parsers remove --all --purge
rune -1 cscli explain --log "$line" --type syslog --crowdsec "$CROWDSEC"
assert_stderr --partial "unable to load parser dump result: no parser found. Please install the appropriate parser and retry"
}
@test 'Allow variable expansion and literal $ characters in passwords' {

16
test/bats/02_nolapi.bats
View file

@ -24,21 +24,23 @@ teardown() {
#----------
@test "test without -no-api flag" {
rune -124 timeout 2s "${CROWDSEC}"
# from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124.
config_set '.common.log_media="stdout"'
rune -0 wait-for \
--err "CrowdSec Local API listening" \
"${CROWDSEC}"
}
@test "crowdsec should not run without LAPI (-no-api flag)" {
# really needs 4 secs on slow boxes
rune -1 timeout 4s "${CROWDSEC}" -no-api
config_set '.common.log_media="stdout"'
rune -1 wait-for "${CROWDSEC}" -no-api
}
@test "crowdsec should not run without LAPI (no api.server in configuration file)" {
config_disable_lapi
config_log_stderr
# really needs 4 secs on slow boxes
rune -1 timeout 4s "${CROWDSEC}"
assert_stderr --partial "crowdsec local API is disabled"
rune -0 wait-for \
--err "crowdsec local API is disabled" \
"${CROWDSEC}"
}
@test "capi status shouldn't be ok without api.server" {

17
test/bats/03_noagent.bats
View file

@ -23,20 +23,25 @@ teardown() {
#----------
@test "with agent: test without -no-cs flag" {
rune -124 timeout 2s "${CROWDSEC}"
# from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124.
config_set '.common.log_media="stdout"'
rune -0 wait-for \
--err "Starting processing data" \
"${CROWDSEC}"
}
@test "no agent: crowdsec LAPI should run (-no-cs flag)" {
rune -124 timeout 2s "${CROWDSEC}" -no-cs
config_set '.common.log_media="stdout"'
rune -0 wait-for \
--err "CrowdSec Local API listening" \
"${CROWDSEC}" -no-cs
}
@test "no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" {
config_disable_agent
config_log_stderr
rune -124 timeout 2s "${CROWDSEC}"
assert_stderr --partial "crowdsec agent is disabled"
rune -0 wait-for \
--err "crowdsec agent is disabled" \
"${CROWDSEC}"
}
@test "no agent: cscli config show" {

4
test/bats/04_capi.bats
View file

@ -22,6 +22,10 @@ setup() {
@test "cscli capi status" {
config_enable_capi
rune -0 cscli capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX
rune -1 cscli capi status
assert_stderr --partial "no scenarios installed, abort"
rune -0 cscli scenarios install crowdsecurity/ssh-bf
rune -0 cscli capi status
assert_stderr --partial "Loaded credentials from"
assert_stderr --partial "Trying to authenticate with username"

Some files were not shown because too many files have changed in this diff Show more