beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

cscli: silence cwhub logger for non-hub related commands (#2675)

Browse source
This commit is contained in:
mmetc 2023-12-19 17:20:09 +01:00 committed by GitHub
parent fd22bb5ec2
commit 6e34d609b7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 156 additions and 150 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec-cli/capi.go
View file

@ -155,7 +155,7 @@ func (cli cliCapi) NewStatusCmd() *cobra.Command {
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err) return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return err return err
} }

4
cmd/crowdsec-cli/config_backup.go
View file

@ -9,15 +9,15 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
) )
func backupHub(dirPath string) error { func backupHub(dirPath string) error {
var itemDirectory string var itemDirectory string
var upstreamParsers []string var upstreamParsers []string
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return err return err
} }

2
cmd/crowdsec-cli/config_restore.go
View file

@ -22,7 +22,7 @@ type OldAPICfg struct {
} }
func restoreHub(dirPath string) error { func restoreHub(dirPath string) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), nil)
if err != nil { if err != nil {
return err return err
} }

5
cmd/crowdsec-cli/console.go
View file

@ -17,12 +17,11 @@ import (
"github.com/crowdsecurity/go-cs-lib/ptr" "github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version" "github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
) )
func NewConsoleCmd() *cobra.Command { func NewConsoleCmd() *cobra.Command {
@ -70,7 +69,7 @@ After running this command your will need to validate the enrollment in the weba
return fmt.Errorf("could not parse CAPI URL: %s", err) return fmt.Errorf("could not parse CAPI URL: %s", err)
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return err return err
} }

6
cmd/crowdsec-cli/hub.go
View file

@ -50,7 +50,7 @@ func (cli cliHub) list(cmd *cobra.Command, args []string) error {
return err return err
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }
@ -100,7 +100,7 @@ func (cli cliHub) update(cmd *cobra.Command, args []string) error {
remote := require.RemoteHub(csConfig) remote := require.RemoteHub(csConfig)
// don't use require.Hub because if there is no index file, it would fail // don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.NewHub(local, remote, true) hub, err := cwhub.NewHub(local, remote, true, log.StandardLogger())
if err != nil { if err != nil {
return fmt.Errorf("failed to update hub: %w", err) return fmt.Errorf("failed to update hub: %w", err)
} }
@ -135,7 +135,7 @@ func (cli cliHub) upgrade(cmd *cobra.Command, args []string) error {
return err return err
} }
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }

4
cmd/crowdsec-cli/item_suggest.go
View file

@ -37,7 +37,7 @@ func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
} }
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return nil, cobra.ShellCompDirectiveDefault return nil, cobra.ShellCompDirectiveDefault
} }
@ -56,7 +56,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
} }
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return nil, cobra.ShellCompDirectiveDefault return nil, cobra.ShellCompDirectiveDefault
} }

10
cmd/crowdsec-cli/itemcli.go
View file

@ -78,7 +78,7 @@ func (cli cliItem) Install(cmd *cobra.Command, args []string) error {
return err return err
} }
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }
@ -161,7 +161,7 @@ func (cli cliItem) Remove(cmd *cobra.Command, args []string) error {
return err return err
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }
@ -273,7 +273,7 @@ func (cli cliItem) Upgrade(cmd *cobra.Command, args []string) error {
return err return err
} }
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }
@ -387,7 +387,7 @@ func (cli cliItem) Inspect(cmd *cobra.Command, args []string) error {
remote = require.RemoteHub(csConfig) remote = require.RemoteHub(csConfig)
} }
hub, err := require.Hub(csConfig, remote) hub, err := require.Hub(csConfig, remote, log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }
@ -468,7 +468,7 @@ func (cli cliItem) List(cmd *cobra.Command, args []string) error {
return err return err
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }

10
cmd/crowdsec-cli/lapi.go
View file

@ -37,7 +37,7 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
return fmt.Errorf("parsing api url: %w", err) return fmt.Errorf("parsing api url: %w", err)
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return err return err
} }
@ -270,7 +270,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`, `,
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return err return err
} }
@ -308,7 +308,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
Short: "List context to send with alerts", Short: "List context to send with alerts",
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error { RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return err return err
} }
@ -355,7 +355,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
return fmt.Errorf("failed to init expr helpers: %w", err) return fmt.Errorf("failed to init expr helpers: %w", err)
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
return err return err
} }
@ -422,7 +422,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
cmdContext.AddCommand(cmdContextDetect) cmdContext.AddCommand(cmdContextDetect)
cmdContextDelete := &cobra.Command{ cmdContextDelete := &cobra.Command{
Use: "delete", Use: "delete",
DisableAutoGenTag: true, DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error { RunE: func(_ *cobra.Command, _ []string) error {
filePath := csConfig.Crowdsec.ConsoleContextPath filePath := csConfig.Crowdsec.ConsoleContextPath

16
cmd/crowdsec-cli/require/require.go
View file

@ -2,6 +2,9 @@ package require
import ( import (
"fmt" "fmt"
"io"
"github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -71,8 +74,8 @@ func Notifications(c *csconfig.Config) error {
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg { func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// set branch in config, and log if necessary // set branch in config, and log if necessary
branch := HubBranch(c) branch := HubBranch(c)
remote := &cwhub.RemoteHubCfg { remote := &cwhub.RemoteHubCfg{
Branch: branch, Branch: branch,
URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s", URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s", // URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
IndexPath: ".index.json", IndexPath: ".index.json",
@ -83,14 +86,19 @@ func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items. // Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations. // If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) { func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
local := c.Hub local := c.Hub
if local == nil { if local == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub") return nil, fmt.Errorf("you must configure cli before interacting with hub")
} }
hub, err := cwhub.NewHub(local, remote, false) if logger == nil {
logger = logrus.New()
logger.SetOutput(io.Discard)
}
hub, err := cwhub.NewHub(local, remote, false, logger)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err) return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
} }

5
cmd/crowdsec-cli/setup.go
View file

@ -11,10 +11,9 @@ import (
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup" "github.com/crowdsecurity/crowdsec/pkg/setup"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
) )
// NewSetupCmd defines the "cscli setup" command. // NewSetupCmd defines the "cscli setup" command.
@ -305,7 +304,7 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return fmt.Errorf("while reading file %s: %w", fromFile, err) return fmt.Errorf("while reading file %s: %w", fromFile, err)
} }
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig)) hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil { if err != nil {
return err return err
} }

5
cmd/crowdsec-cli/simulation.go
View file

@ -13,7 +13,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
) )
type cliSimulation struct {} type cliSimulation struct{}
func NewCLISimulation() *cliSimulation { func NewCLISimulation() *cliSimulation {
return &cliSimulation{} return &cliSimulation{}
@ -61,7 +61,7 @@ func (cli cliSimulation) NewEnableCmd() *cobra.Command {
Example: `cscli simulation enable`, Example: `cscli simulation enable`,
DisableAutoGenTag: true, DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -267,4 +267,3 @@ func simulationStatus() error {
} }
return nil return nil
} }

4
cmd/crowdsec-cli/support.go
View file

@ -238,7 +238,7 @@ func collectAcquisitionConfig() map[string][]byte {
return ret return ret
} }
type cliSupport struct {} type cliSupport struct{}
func NewCLISupport() *cliSupport { func NewCLISupport() *cliSupport {
return &cliSupport{} return &cliSupport{}
@ -316,7 +316,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
skipAgent = true skipAgent = true
} }
hub, err := require.Hub(csConfig, nil) hub, err := require.Hub(csConfig, nil, nil)
if err != nil { if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true skipHub = true

4
cmd/crowdsec/serve.go
View file

@ -77,7 +77,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
} }
if !cConfig.DisableAgent { if !cConfig.DisableAgent {
hub, err := cwhub.NewHub(cConfig.Hub, nil, false) hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
if err != nil { if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err) return nil, fmt.Errorf("while loading hub index: %w", err)
} }
@ -348,7 +348,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
} }
if !cConfig.DisableAgent { if !cConfig.DisableAgent {
hub, err := cwhub.NewHub(cConfig.Hub, nil, false) hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
if err != nil { if err != nil {
return fmt.Errorf("while loading hub index: %w", err) return fmt.Errorf("while loading hub index: %w", err)
} }

2
pkg/cwhub/cwhub_test.go
View file

@ -61,7 +61,7 @@ func testHub(t *testing.T, update bool) *Hub {
IndexPath: ".index.json", IndexPath: ".index.json",
} }
hub, err := NewHub(local, remote, update) hub, err := NewHub(local, remote, update, log.StandardLogger())
require.NoError(t, err) require.NoError(t, err)
return hub return hub

23
pkg/cwhub/dataset.go
View file

@ -8,7 +8,7 @@ import (
"os" "os"
"time" "time"
log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
@ -21,8 +21,6 @@ type DataSet struct {
// downloadFile downloads a file and writes it to disk, with no hash verification. // downloadFile downloads a file and writes it to disk, with no hash verification.
func downloadFile(url string, destPath string) error { func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath)
resp, err := hubClient.Get(url) resp, err := hubClient.Get(url)
if err != nil { if err != nil {
return fmt.Errorf("while downloading %s: %w", url, err) return fmt.Errorf("while downloading %s: %w", url, err)
@ -56,26 +54,26 @@ func downloadFile(url string, destPath string) error {
// if the local file doesn't exist, update. // if the local file doesn't exist, update.
// if the remote is newer than the local file, update. // if the remote is newer than the local file, update.
// if the remote has no modification date, but local file has been modified > a week ago, update. // if the remote has no modification date, but local file has been modified > a week ago, update.
func needsUpdate(destPath string, url string) bool { func needsUpdate(destPath string, url string, logger *logrus.Logger) bool {
fileInfo, err := os.Stat(destPath) fileInfo, err := os.Stat(destPath)
switch { switch {
case os.IsNotExist(err): case os.IsNotExist(err):
return true return true
case err != nil: case err != nil:
log.Errorf("while getting %s: %s", destPath, err) logger.Errorf("while getting %s: %s", destPath, err)
return true return true
} }
resp, err := hubClient.Head(url) resp, err := hubClient.Head(url)
if err != nil { if err != nil {
log.Errorf("while getting %s: %s", url, err) logger.Errorf("while getting %s: %s", url, err)
// Head failed, Get would likely fail too -> no update // Head failed, Get would likely fail too -> no update
return false return false
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
log.Errorf("bad http code %d for %s", resp.StatusCode, url) logger.Errorf("bad http code %d for %s", resp.StatusCode, url)
return false return false
} }
@ -89,19 +87,19 @@ func needsUpdate(destPath string, url string) bool {
remoteLastModified := resp.Header.Get("Last-Modified") remoteLastModified := resp.Header.Get("Last-Modified")
if remoteLastModified == "" { if remoteLastModified == "" {
if localIsOld { if localIsOld {
log.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife) logger.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife)
} }
return localIsOld return localIsOld
} }
lastAvailable, err := time.Parse(time.RFC1123, remoteLastModified) lastAvailable, err := time.Parse(time.RFC1123, remoteLastModified)
if err != nil { if err != nil {
log.Warningf("while parsing last modified date for %s: %s", url, err) logger.Warningf("while parsing last modified date for %s: %s", url, err)
return localIsOld return localIsOld
} }
if lastModify.Before(lastAvailable) { if lastModify.Before(lastAvailable) {
log.Infof("new version available, updating %s", destPath) logger.Infof("new version available, updating %s", destPath)
return true return true
} }
@ -109,7 +107,7 @@ func needsUpdate(destPath string, url string) bool {
} }
// downloadDataSet downloads all the data files for an item. // downloadDataSet downloads all the data files for an item.
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error { func downloadDataSet(dataFolder string, force bool, reader io.Reader, logger *logrus.Logger) error {
dec := yaml.NewDecoder(reader) dec := yaml.NewDecoder(reader)
for { for {
@ -129,7 +127,8 @@ func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
return err return err
} }
if force || needsUpdate(destPath, dataS.SourceURL) { if force || needsUpdate(destPath, dataS.SourceURL, logger) {
logger.Debugf("downloading %s in %s", dataS.SourceURL, destPath)
if err := downloadFile(dataS.SourceURL, destPath); err != nil { if err := downloadFile(dataS.SourceURL, destPath); err != nil {
return fmt.Errorf("while getting data: %w", err) return fmt.Errorf("while getting data: %w", err)
} }

24
pkg/cwhub/hub.go
View file

@ -4,11 +4,12 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"os" "os"
"path" "path"
"strings" "strings"
log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"slices" "slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
@ -20,6 +21,7 @@ type Hub struct {
local *csconfig.LocalHubCfg local *csconfig.LocalHubCfg
remote *RemoteHubCfg remote *RemoteHubCfg
Warnings []string // Warnings encountered during sync Warnings []string // Warnings encountered during sync
logger *logrus.Logger
} }
// GetDataDir returns the data directory, where data sets are installed. // GetDataDir returns the data directory, where data sets are installed.
@ -30,14 +32,20 @@ func (h *Hub) GetDataDir() string {
// NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state. // NewHub returns a new Hub instance with local and (optionally) remote configuration, and syncs the local state.
// If updateIndex is true, the local index file is updated from the remote before reading the state of the items. // If updateIndex is true, the local index file is updated from the remote before reading the state of the items.
// All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set. // All download operations (including updateIndex) return ErrNilRemoteHub if the remote configuration is not set.
func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool) (*Hub, error) { func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool, logger *logrus.Logger) (*Hub, error) {
if local == nil { if local == nil {
return nil, fmt.Errorf("no hub configuration found") return nil, fmt.Errorf("no hub configuration found")
} }
if logger == nil {
logger = logrus.New()
logger.SetOutput(io.Discard)
}
hub := &Hub{ hub := &Hub{
local: local, local: local,
remote: remote, remote: remote,
logger: logger,
} }
if updateIndex { if updateIndex {
@ -46,7 +54,7 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, updateIndex bool)
} }
} }
log.Debugf("loading hub idx %s", local.HubIndexFile) logger.Debugf("loading hub idx %s", local.HubIndexFile)
if err := hub.parseIndex(); err != nil { if err := hub.parseIndex(); err != nil {
return nil, fmt.Errorf("failed to load index: %w", err) return nil, fmt.Errorf("failed to load index: %w", err)
@ -70,11 +78,11 @@ func (h *Hub) parseIndex() error {
return fmt.Errorf("failed to unmarshal index: %w", err) return fmt.Errorf("failed to unmarshal index: %w", err)
} }
log.Debugf("%d item types in hub index", len(ItemTypes)) h.logger.Debugf("%d item types in hub index", len(ItemTypes))
// Iterate over the different types to complete the struct // Iterate over the different types to complete the struct
for _, itemType := range ItemTypes { for _, itemType := range ItemTypes {
log.Tracef("%s: %d items", itemType, len(h.GetItemMap(itemType))) h.logger.Tracef("%s: %d items", itemType, len(h.GetItemMap(itemType)))
for name, item := range h.GetItemMap(itemType) { for name, item := range h.GetItemMap(itemType) {
item.hub = h item.hub = h
@ -145,10 +153,10 @@ func (h *Hub) updateIndex() error {
oldContent, err := os.ReadFile(h.local.HubIndexFile) oldContent, err := os.ReadFile(h.local.HubIndexFile)
if err != nil { if err != nil {
if !os.IsNotExist(err) { if !os.IsNotExist(err) {
log.Warningf("failed to read hub index: %s", err) h.logger.Warningf("failed to read hub index: %s", err)
} }
} else if bytes.Equal(body, oldContent) { } else if bytes.Equal(body, oldContent) {
log.Info("hub index is up to date") h.logger.Info("hub index is up to date")
return nil return nil
} }
@ -156,7 +164,7 @@ func (h *Hub) updateIndex() error {
return fmt.Errorf("failed to write hub index: %w", err) return fmt.Errorf("failed to write hub index: %w", err)
} }
log.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body)) h.logger.Infof("Wrote index to %s, %d bytes", h.local.HubIndexFile, len(body))
return nil return nil
} }

2
pkg/cwhub/hub_test.go
View file

@ -19,7 +19,7 @@ func TestInitHubUpdate(t *testing.T) {
IndexPath: ".index.json", IndexPath: ".index.json",
} }
_, err := NewHub(hub.local, remote, true) _, err := NewHub(hub.local, remote, true, nil)
require.NoError(t, err) require.NoError(t, err)
} }

19
pkg/cwhub/item.go
View file

@ -7,7 +7,6 @@ import (
"github.com/Masterminds/semver/v3" "github.com/Masterminds/semver/v3"
"github.com/enescakir/emoji" "github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
"slices" "slices"
) )
@ -281,43 +280,43 @@ func (i *Item) logMissingSubItems() {
for _, subName := range i.Parsers { for _, subName := range i.Parsers {
if i.hub.GetItem(PARSERS, subName) == nil { if i.hub.GetItem(PARSERS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name) i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name)
} }
} }
for _, subName := range i.Scenarios { for _, subName := range i.Scenarios {
if i.hub.GetItem(SCENARIOS, subName) == nil { if i.hub.GetItem(SCENARIOS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name) i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name)
} }
} }
for _, subName := range i.PostOverflows { for _, subName := range i.PostOverflows {
if i.hub.GetItem(POSTOVERFLOWS, subName) == nil { if i.hub.GetItem(POSTOVERFLOWS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name) i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name)
} }
} }
for _, subName := range i.Contexts { for _, subName := range i.Contexts {
if i.hub.GetItem(CONTEXTS, subName) == nil { if i.hub.GetItem(CONTEXTS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name) i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name)
} }
} }
for _, subName := range i.AppsecConfigs { for _, subName := range i.AppsecConfigs {
if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil { if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name) i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name)
} }
} }
for _, subName := range i.AppsecRules { for _, subName := range i.AppsecRules {
if i.hub.GetItem(APPSEC_RULES, subName) == nil { if i.hub.GetItem(APPSEC_RULES, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name) i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, APPSEC_RULES, i.Name)
} }
} }
for _, subName := range i.Collections { for _, subName := range i.Collections {
if i.hub.GetItem(COLLECTIONS, subName) == nil { if i.hub.GetItem(COLLECTIONS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name) i.hub.logger.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name)
} }
} }
} }
@ -410,7 +409,7 @@ func (i *Item) validPath(dirName, fileName string) bool {
} }
// FQName returns the fully qualified name of the item (ie. parsers:crowdsecurity/apache2-logs). // FQName returns the fully qualified name of the item (ie. parsers:crowdsecurity/apache2-logs).
func (i *Item) FQName () string { func (i *Item) FQName() string {
return fmt.Sprintf("%s:%s", i.Type, i.Name) return fmt.Sprintf("%s:%s", i.Type, i.Name)
} }
@ -433,7 +432,7 @@ func (i *Item) addTaint(sub *Item) {
i.State.TaintedBy[idx] = taintedBy i.State.TaintedBy[idx] = taintedBy
log.Debugf("%s is tainted by %s", i.Name, taintedBy) i.hub.logger.Debugf("%s is tainted by %s", i.Name, taintedBy)
// propagate the taint to the ancestors // propagate the taint to the ancestors

12
pkg/cwhub/iteminstall.go
View file

@ -2,8 +2,6 @@ package cwhub
import ( import (
"fmt" "fmt"
log "github.com/sirupsen/logrus"
) )
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items. // enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
@ -19,7 +17,7 @@ func (i *Item) enable() error {
// if it's a collection, check sub-items even if the collection file itself is up-to-date // if it's a collection, check sub-items even if the collection file itself is up-to-date
if i.State.UpToDate && !i.HasSubItems() { if i.State.UpToDate && !i.HasSubItems() {
log.Tracef("%s is installed and up-to-date, skip.", i.Name) i.hub.logger.Tracef("%s is installed and up-to-date, skip.", i.Name)
return nil return nil
} }
} }
@ -34,7 +32,7 @@ func (i *Item) enable() error {
return err return err
} }
log.Infof("Enabled %s: %s", i.Type, i.Name) i.hub.logger.Infof("Enabled %s: %s", i.Type, i.Name)
i.State.Installed = true i.State.Installed = true
return nil return nil
@ -43,7 +41,7 @@ func (i *Item) enable() error {
// Install installs the item from the hub, downloading it if needed. // Install installs the item from the hub, downloading it if needed.
func (i *Item) Install(force bool, downloadOnly bool) error { func (i *Item) Install(force bool, downloadOnly bool) error {
if downloadOnly && i.State.Downloaded && i.State.UpToDate { if downloadOnly && i.State.Downloaded && i.State.UpToDate {
log.Infof("%s is already downloaded and up-to-date", i.Name) i.hub.logger.Infof("%s is already downloaded and up-to-date", i.Name)
if !force { if !force {
return nil return nil
@ -56,7 +54,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
} }
if downloadOnly { if downloadOnly {
log.Infof("Downloaded %s to %s", i.Name, filePath) i.hub.logger.Infof("Downloaded %s to %s", i.Name, filePath)
return nil return nil
} }
@ -64,7 +62,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
return fmt.Errorf("while enabling %s: %w", i.Name, err) return fmt.Errorf("while enabling %s: %w", i.Name, err)
} }
log.Infof("Enabled %s", i.Name) i.hub.logger.Infof("Enabled %s", i.Name)
return nil return nil
} }

10
pkg/cwhub/itemlink.go
View file

@ -4,8 +4,6 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
log "github.com/sirupsen/logrus"
) )
// createInstallLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir. // createInstallLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
@ -21,7 +19,7 @@ func (i *Item) createInstallLink() error {
} }
if _, err = os.Lstat(dest); !os.IsNotExist(err) { if _, err = os.Lstat(dest); !os.IsNotExist(err) {
log.Infof("%s already exists.", dest) i.hub.logger.Infof("%s already exists.", dest)
return nil return nil
} }
@ -51,7 +49,7 @@ func (i *Item) removeInstallLink() error {
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ... // if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 { if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml) i.hub.logger.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
return fmt.Errorf("%s isn't managed by hub", i.Name) return fmt.Errorf("%s isn't managed by hub", i.Name)
} }
@ -66,7 +64,7 @@ func (i *Item) removeInstallLink() error {
} }
if hubpath != src { if hubpath != src {
log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src) i.hub.logger.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
return fmt.Errorf("%s isn't managed by hub", i.Name) return fmt.Errorf("%s isn't managed by hub", i.Name)
} }
@ -74,7 +72,7 @@ func (i *Item) removeInstallLink() error {
return fmt.Errorf("while removing symlink: %w", err) return fmt.Errorf("while removing symlink: %w", err)
} }
log.Infof("Removed symlink [%s]: %s", i.Name, syml) i.hub.logger.Infof("Removed symlink [%s]: %s", i.Name, syml)
return nil return nil
} }

13
pkg/cwhub/itemremove.go
View file

@ -4,14 +4,13 @@ import (
"fmt" "fmt"
"os" "os"
log "github.com/sirupsen/logrus"
"slices" "slices"
) )
// purge removes the actual config file that was downloaded. // purge removes the actual config file that was downloaded.
func (i *Item) purge() (bool, error) { func (i *Item) purge() (bool, error) {
if !i.State.Downloaded { if !i.State.Downloaded {
log.Debugf("removing %s: not downloaded -- no need to remove", i.Name) i.hub.logger.Debugf("removing %s: not downloaded -- no need to remove", i.Name)
return false, nil return false, nil
} }
@ -22,7 +21,7 @@ func (i *Item) purge() (bool, error) {
if err := os.Remove(src); err != nil { if err := os.Remove(src); err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
log.Debugf("%s doesn't exist, no need to remove", src) i.hub.logger.Debugf("%s doesn't exist, no need to remove", src)
return false, nil return false, nil
} }
@ -30,7 +29,7 @@ func (i *Item) purge() (bool, error) {
} }
i.State.Downloaded = false i.State.Downloaded = false
log.Infof("Removed source file [%s]: %s", i.Name, src) i.hub.logger.Infof("Removed source file [%s]: %s", i.Name, src)
return true, nil return true, nil
} }
@ -68,7 +67,7 @@ func (i *Item) disable(purge bool, force bool) (bool, error) {
// Remove disables the item, optionally removing the downloaded content. // Remove disables the item, optionally removing the downloaded content.
func (i *Item) Remove(purge bool, force bool) (bool, error) { func (i *Item) Remove(purge bool, force bool) (bool, error) {
if i.State.IsLocal() { if i.State.IsLocal() {
log.Warningf("%s is a local item, please delete manually", i.Name) i.hub.logger.Warningf("%s is a local item, please delete manually", i.Name)
return false, nil return false, nil
} }
@ -77,7 +76,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
} }
if !i.State.Installed && !purge { if !i.State.Installed && !purge {
log.Infof("removing %s: not installed -- no need to remove", i.Name) i.hub.logger.Infof("removing %s: not installed -- no need to remove", i.Name)
return false, nil return false, nil
} }
@ -116,7 +115,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
} }
if !slices.Contains(descendants, subParent) { if !slices.Contains(descendants, subParent) {
log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name) i.hub.logger.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
continue continue
} }
} }

37
pkg/cwhub/itemupgrade.go
View file

@ -13,7 +13,6 @@ import (
"path/filepath" "path/filepath"
"github.com/enescakir/emoji" "github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
) )
// Upgrade downloads and applies the last version of the item from the hub. // Upgrade downloads and applies the last version of the item from the hub.
@ -21,7 +20,7 @@ func (i *Item) Upgrade(force bool) (bool, error) {
updated := false updated := false
if i.State.IsLocal() { if i.State.IsLocal() {
log.Infof("not upgrading %s: local item", i.Name) i.hub.logger.Infof("not upgrading %s: local item", i.Name)
return false, nil return false, nil
} }
@ -34,7 +33,7 @@ func (i *Item) Upgrade(force bool) (bool, error) {
} }
if i.State.UpToDate { if i.State.UpToDate {
log.Infof("%s: up-to-date", i.Name) i.hub.logger.Infof("%s: up-to-date", i.Name)
if err := i.DownloadDataIfNeeded(force); err != nil { if err := i.DownloadDataIfNeeded(force); err != nil {
return false, fmt.Errorf("%s: download failed: %w", i.Name, err) return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
@ -52,14 +51,14 @@ func (i *Item) Upgrade(force bool) (bool, error) {
if !i.State.UpToDate { if !i.State.UpToDate {
if i.State.Tainted { if i.State.Tainted {
log.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name) i.hub.logger.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
} }
} else { } else {
// a check on stdout is used while scripting to know if the hub has been upgraded // a check on stdout is used while scripting to know if the hub has been upgraded
// and a configuration reload is required // and a configuration reload is required
// TODO: use a better way to communicate this // TODO: use a better way to communicate this
fmt.Printf("updated %s\n", i.Name) fmt.Printf("updated %s\n", i.Name)
log.Infof("%v %s: updated", emoji.Package, i.Name) i.hub.logger.Infof("%v %s: updated", emoji.Package, i.Name)
updated = true updated = true
} }
@ -68,19 +67,19 @@ func (i *Item) Upgrade(force bool) (bool, error) {
// downloadLatest downloads the latest version of the item to the hub directory. // downloadLatest downloads the latest version of the item to the hub directory.
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) { func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
log.Debugf("Downloading %s %s", i.Type, i.Name) i.hub.logger.Debugf("Downloading %s %s", i.Type, i.Name)
for _, sub := range i.SubItems() { for _, sub := range i.SubItems() {
if !sub.State.Installed && updateOnly && sub.State.Downloaded { if !sub.State.Installed && updateOnly && sub.State.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name) i.hub.logger.Debugf("skipping upgrade of %s: not installed", i.Name)
continue continue
} }
log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly) i.hub.logger.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.State.Installed, updateOnly)
// recurse as it's a collection // recurse as it's a collection
if sub.HasSubItems() { if sub.HasSubItems() {
log.Tracef("collection, recurse") i.hub.logger.Tracef("collection, recurse")
if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil { if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil {
return "", fmt.Errorf("while downloading %s: %w", sub.Name, err) return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
@ -103,7 +102,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
} }
if !i.State.Installed && updateOnly && i.State.Downloaded { if !i.State.Installed && updateOnly && i.State.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name) i.hub.logger.Debugf("skipping upgrade of %s: not installed", i.Name)
return "", nil return "", nil
} }
@ -144,8 +143,8 @@ func (i *Item) FetchLatest() ([]byte, string, error) {
meow := hex.EncodeToString(hash.Sum(nil)) meow := hex.EncodeToString(hash.Sum(nil))
if meow != i.Versions[i.Version].Digest { if meow != i.Versions[i.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'") i.hub.logger.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest) i.hub.logger.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
return nil, "", fmt.Errorf("invalid download hash for %s", i.Name) return nil, "", fmt.Errorf("invalid download hash for %s", i.Name)
} }
@ -161,13 +160,13 @@ func (i *Item) download(overwrite bool) (string, error) {
// if user didn't --force, don't overwrite local, tainted, up-to-date files // if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite { if !overwrite {
if i.State.Tainted { if i.State.Tainted {
log.Debugf("%s: tainted, not updated", i.Name) i.hub.logger.Debugf("%s: tainted, not updated", i.Name)
return "", nil return "", nil
} }
if i.State.UpToDate { if i.State.UpToDate {
// We still have to check if data files are present // We still have to check if data files are present
log.Debugf("%s: up-to-date, not updated", i.Name) i.hub.logger.Debugf("%s: up-to-date, not updated", i.Name)
} }
} }
@ -192,10 +191,10 @@ func (i *Item) download(overwrite bool) (string, error) {
// check actual file // check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) { if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s: overwrite", i.Name) i.hub.logger.Warningf("%s: overwrite", i.Name)
log.Debugf("target: %s", finalPath) i.hub.logger.Debugf("target: %s", finalPath)
} else { } else {
log.Infof("%s: OK", i.Name) i.hub.logger.Infof("%s: OK", i.Name)
} }
if err = os.WriteFile(finalPath, body, 0o644); err != nil { if err = os.WriteFile(finalPath, body, 0o644); err != nil {
@ -206,7 +205,7 @@ func (i *Item) download(overwrite bool) (string, error) {
i.State.Tainted = false i.State.Tainted = false
i.State.UpToDate = true i.State.UpToDate = true
if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body), i.hub.logger); err != nil {
return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err) return "", fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
} }
@ -227,7 +226,7 @@ func (i *Item) DownloadDataIfNeeded(force bool) error {
defer itemFile.Close() defer itemFile.Close()
if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile); err != nil { if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile, i.hub.logger); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
} }

8
pkg/cwhub/itemupgrade_test.go
View file

@ -40,7 +40,7 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
IndexPath: ".index.json", IndexPath: ".index.json",
} }
hub, err := NewHub(hub.local, remote, true) hub, err := NewHub(hub.local, remote, true, nil)
require.NoError(t, err, "failed to download index: %s", err) require.NoError(t, err, "failed to download index: %s", err)
hub = getHubOrFail(t, hub.local, remote) hub = getHubOrFail(t, hub.local, remote)
@ -102,7 +102,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed) require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate) require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate)
hub, err = NewHub(hub.local, remote, true) hub, err = NewHub(hub.local, remote, true, nil)
require.NoError(t, err, "failed to download index: %s", err) require.NoError(t, err, "failed to download index: %s", err)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection") item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
@ -116,7 +116,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test. // getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub { func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
hub, err := NewHub(local, remote, false) hub, err := NewHub(local, remote, false, nil)
require.NoError(t, err, "failed to load hub index") require.NoError(t, err, "failed to load hub index")
return hub return hub
@ -169,7 +169,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
// we just removed. Nor should it install the newly added scenario // we just removed. Nor should it install the newly added scenario
pushUpdateToCollectionInHub() pushUpdateToCollectionInHub()
hub, err = NewHub(hub.local, remote, true) hub, err = NewHub(hub.local, remote, true, nil)
require.NoError(t, err, "failed to download index: %s", err) require.NoError(t, err, "failed to download index: %s", err)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed) require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)

58
pkg/cwhub/sync.go
View file

@ -11,7 +11,7 @@ import (
"strings" "strings"
"github.com/Masterminds/semver/v3" "github.com/Masterminds/semver/v3"
log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
"slices" "slices"
) )
@ -21,17 +21,17 @@ func isYAMLFileName(path string) bool {
} }
// linkTarget returns the target of a symlink, or empty string if it's dangling. // linkTarget returns the target of a symlink, or empty string if it's dangling.
func linkTarget(path string) (string, error) { func linkTarget(path string, logger *logrus.Logger) (string, error) {
hubpath, err := os.Readlink(path) hubpath, err := os.Readlink(path)
if err != nil { if err != nil {
return "", fmt.Errorf("unable to read symlink: %s", path) return "", fmt.Errorf("unable to read symlink: %s", path)
} }
log.Tracef("symlink %s -> %s", path, hubpath) logger.Tracef("symlink %s -> %s", path, hubpath)
_, err = os.Lstat(hubpath) _, err = os.Lstat(hubpath)
if os.IsNotExist(err) { if os.IsNotExist(err) {
log.Warningf("link target does not exist: %s -> %s", path, hubpath) logger.Warningf("link target does not exist: %s -> %s", path, hubpath)
return "", nil return "", nil
} }
@ -63,7 +63,7 @@ type itemFileInfo struct {
fauthor string fauthor string
} }
func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) { func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo, error) {
var ret *itemFileInfo var ret *itemFileInfo
hubDir := h.local.HubDir hubDir := h.local.HubDir
@ -71,11 +71,11 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
subs := strings.Split(path, string(os.PathSeparator)) subs := strings.Split(path, string(os.PathSeparator))
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir) logger.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
log.Tracef("subs:%v", subs) logger.Tracef("subs:%v", subs)
// we're in hub (~/.hub/hub/) // we're in hub (~/.hub/hub/)
if strings.HasPrefix(path, hubDir) { if strings.HasPrefix(path, hubDir) {
log.Tracef("in hub dir") logger.Tracef("in hub dir")
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/scenarios/crowdsec/ssh_bf.yaml
@ -92,7 +92,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
ftype: subs[len(subs)-4], ftype: subs[len(subs)-4],
} }
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/... } else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir") logger.Tracef("in install dir")
if len(subs) < 3 { if len(subs) < 3 {
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
} }
@ -111,7 +111,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
} }
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) logger.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
if !slices.Contains(ItemTypes, ret.stage) { if !slices.Contains(ItemTypes, ret.stage) {
@ -122,7 +122,7 @@ func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
ret.stage = "" ret.stage = ""
} }
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) logger.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
return ret, nil return ret, nil
} }
@ -194,7 +194,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
hubpath := "" hubpath := ""
if err != nil { if err != nil {
log.Debugf("while syncing hub dir: %s", err) h.logger.Debugf("while syncing hub dir: %s", err)
// there is a path error, we ignore the file // there is a path error, we ignore the file
return nil return nil
} }
@ -210,17 +210,17 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
return nil return nil
} }
info, err := h.getItemFileInfo(path) info, err := h.getItemFileInfo(path, h.logger)
if err != nil { if err != nil {
return err return err
} }
// non symlinks are local user files or hub files // non symlinks are local user files or hub files
if f.Type()&os.ModeSymlink == 0 { if f.Type()&os.ModeSymlink == 0 {
log.Tracef("%s is not a symlink", path) h.logger.Tracef("%s is not a symlink", path)
if !info.inhub { if !info.inhub {
log.Tracef("%s is a local file, skip", path) h.logger.Tracef("%s is a local file, skip", path)
item, err := newLocalItem(h, path, info) item, err := newLocalItem(h, path, info)
if err != nil { if err != nil {
@ -232,7 +232,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
return nil return nil
} }
} else { } else {
hubpath, err = linkTarget(path) hubpath, err = linkTarget(path, h.logger)
if err != nil { if err != nil {
return err return err
} }
@ -245,7 +245,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
} }
// try to find which configuration item it is // try to find which configuration item it is
log.Tracef("check [%s] of %s", info.fname, info.ftype) h.logger.Tracef("check [%s] of %s", info.fname, info.ftype)
for _, item := range h.GetItemMap(info.ftype) { for _, item := range h.GetItemMap(info.ftype) {
if info.fname != item.FileName { if info.fname != item.FileName {
@ -274,7 +274,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
} }
if path == src { if path == src {
log.Tracef("marking %s as downloaded", item.Name) h.logger.Tracef("marking %s as downloaded", item.Name)
item.State.Downloaded = true item.State.Downloaded = true
} }
} else if !hasPathSuffix(hubpath, item.RemotePath) { } else if !hasPathSuffix(hubpath, item.RemotePath) {
@ -291,7 +291,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
return nil return nil
} }
log.Infof("Ignoring file %s of type %s", path, info.ftype) h.logger.Infof("Ignoring file %s of type %s", path, info.ftype)
return nil return nil
} }
@ -305,15 +305,15 @@ func (i *Item) checkSubItemVersions() []string {
} }
if i.versionStatus() != versionUpToDate { if i.versionStatus() != versionUpToDate {
log.Debugf("%s dependencies not checked: not up-to-date", i.Name) i.hub.logger.Debugf("%s dependencies not checked: not up-to-date", i.Name)
return warn return warn
} }
// ensure all the sub-items are installed, or tag the parent as tainted // ensure all the sub-items are installed, or tag the parent as tainted
log.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed) i.hub.logger.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed)
for _, sub := range i.SubItems() { for _, sub := range i.SubItems() {
log.Tracef("check %s installed:%t", sub.Name, sub.State.Installed) i.hub.logger.Tracef("check %s installed:%t", sub.Name, sub.State.Installed)
if !i.State.Installed { if !i.State.Installed {
continue continue
@ -351,7 +351,7 @@ func (i *Item) checkSubItemVersions() []string {
continue continue
} }
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate) i.hub.logger.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate)
} }
return warn return warn
@ -365,13 +365,13 @@ func (h *Hub) syncDir(dir string) error {
// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ... // i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
if err != nil { if err != nil {
log.Errorf("failed %s: %s", cpath, err) h.logger.Errorf("failed %s: %s", cpath, err)
continue continue
} }
// explicit check for non existing directory, avoid spamming log.Debug // explicit check for non existing directory, avoid spamming log.Debug
if _, err = os.Stat(cpath); os.IsNotExist(err) { if _, err = os.Stat(cpath); os.IsNotExist(err) {
log.Tracef("directory %s doesn't exist, skipping", cpath) h.logger.Tracef("directory %s doesn't exist, skipping", cpath)
continue continue
} }
@ -454,7 +454,7 @@ func (h *Hub) localSync() error {
} }
} }
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions) h.logger.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
} }
h.Warnings = removeDuplicates(warnings) h.Warnings = removeDuplicates(warnings)
@ -491,7 +491,7 @@ func (i *Item) setVersionState(path string, inhub bool) error {
} }
if i.State.LocalVersion == "?" { if i.State.LocalVersion == "?" {
log.Tracef("got tainted match for %s: %s", i.Name, path) i.hub.logger.Tracef("got tainted match for %s: %s", i.Name, path)
if !inhub { if !inhub {
i.State.LocalPath = path i.State.LocalPath = path
@ -509,7 +509,7 @@ func (i *Item) setVersionState(path string, inhub bool) error {
i.State.Downloaded = true i.State.Downloaded = true
if !inhub { if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version) i.hub.logger.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version)
i.State.LocalPath = path i.State.LocalPath = path
i.State.Tainted = false i.State.Tainted = false
// if we're walking the hub, present file doesn't means installed file // if we're walking the hub, present file doesn't means installed file
@ -517,7 +517,7 @@ func (i *Item) setVersionState(path string, inhub bool) error {
} }
if i.State.LocalVersion == i.Version { if i.State.LocalVersion == i.Version {
log.Tracef("%s is up-to-date", i.Name) i.hub.logger.Tracef("%s is up-to-date", i.Name)
i.State.UpToDate = true i.State.UpToDate = true
} }

4
pkg/hubtest/hubtest.go
View file

@ -93,7 +93,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
InstallDataDir: HubTestPath, InstallDataDir: HubTestPath,
} }
hub, err := cwhub.NewHub(local, nil, false) hub, err := cwhub.NewHub(local, nil, false, nil)
if err != nil { if err != nil {
return HubTest{}, fmt.Errorf("unable to load hub: %s", err) return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
} }
@ -126,7 +126,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string, isAppsecT
InstallDataDir: HubTestPath, InstallDataDir: HubTestPath,
} }
hub, err := cwhub.NewHub(local, nil, false) hub, err := cwhub.NewHub(local, nil, false, nil)
if err != nil { if err != nil {
return HubTest{}, fmt.Errorf("unable to load hub: %s", err) return HubTest{}, fmt.Errorf("unable to load hub: %s", err)
} }

2
pkg/hubtest/hubtest_item.go
View file

@ -495,7 +495,7 @@ func (t *HubTestItem) InstallHub() error {
} }
// load installed hub // load installed hub
hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false) hub, err := cwhub.NewHub(t.RuntimeHubConfig, nil, false, nil)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }

15
pkg/leakybucket/buckets_test.go
View file

@ -14,15 +14,16 @@ import (
"testing" "testing"
"time" "time"
"github.com/davecgh/go-spew/spew"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
yaml "gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/davecgh/go-spew/spew"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
yaml "gopkg.in/yaml.v2"
) )
type TestFile struct { type TestFile struct {
@ -39,12 +40,12 @@ func TestBucket(t *testing.T) {
testdata := "./tests" testdata := "./tests"
hubCfg := &csconfig.LocalHubCfg{ hubCfg := &csconfig.LocalHubCfg{
HubDir: filepath.Join(testdata, "hub"), HubDir: filepath.Join(testdata, "hub"),
HubIndexFile: filepath.Join(testdata, "hub", "index.json"), HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
InstallDataDir: testdata, InstallDataDir: testdata,
} }
hub, err := cwhub.NewHub(hubCfg, nil, false) hub, err := cwhub.NewHub(hubCfg, nil, false, nil)
if err != nil { if err != nil {
t.Fatalf("failed to init hub: %s", err) t.Fatalf("failed to init hub: %s", err)
} }