beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

merge hub-1.5.6 branch

Browse source
This commit is contained in:
Sebastien Blot 2023-11-10 17:36:17 +01:00
commit d6f9bbc0c3
No known key found for this signature in database
GPG key ID: DFC2902F40449F6A
38 changed files with 506 additions and 645 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
cmd/crowdsec-cli/config_restore.go
View file

@ -21,31 +21,6 @@ type OldAPICfg struct {
Password string `json:"password"`
}
// it's a rip of the cli version, but in silent-mode
// XXX: redundant, should call InstallItem
func silentInstallItem(hub *cwhub.Hub, name, obtype string) (string, error) {
var item = hub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
err := hub.DownloadLatest(item, false, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
if err = hub.AddItem(*item); err != nil {
return "", err
}
err = hub.EnableItem(item)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
}
if err := hub.AddItem(*item); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", item.Name), nil
}
func restoreHub(dirPath string) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
@ -70,13 +45,14 @@ func restoreHub(dirPath string) error {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silentInstallItem(hub, toinstall, itype)
item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
err := item.Install(false, false)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}

37
cmd/crowdsec-cli/hub.go
View file

@ -24,13 +24,6 @@ cscli hub update
cscli hub upgrade`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
return nil
},
}
cmdHub.AddCommand(NewHubListCmd())
@ -53,13 +46,11 @@ func runHubList(cmd *cobra.Command, args []string) error {
return err
}
// use LocalSync to get warnings about tainted / outdated items
warn, _ := hub.LocalSync()
for _, v := range warn {
for _, v := range hub.Warnings {
log.Info(v)
}
for line := range hub.ItemStats() {
for _, line := range hub.ItemStats() {
log.Info(line)
}
@ -96,9 +87,7 @@ func runHubUpdate(cmd *cobra.Command, args []string) error {
return fmt.Errorf("failed to update hub: %w", err)
}
// use LocalSync to get warnings about tainted / outdated items
warn, _ := hub.LocalSync()
for _, v := range warn {
for _, v := range hub.Warnings {
log.Info(v)
}
@ -114,14 +103,7 @@ Fetches the .index.json file from the hub, containing the list of available conf
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
return nil
},
RunE: runHubUpdate,
RunE: runHubUpdate,
}
return cmdHubUpdate
@ -149,7 +131,7 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := hub.UpgradeItem(itemType, item.Name, force)
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
@ -172,14 +154,7 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
return nil
},
RunE: runHubUpgrade,
RunE: runHubUpgrade,
}
flags := cmdHubUpgrade.Flags()

2
cmd/crowdsec-cli/hubtest.go
View file

@ -136,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
}
configFilePath := filepath.Join(testPath, "config.yaml")
fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
fd, err := os.Create(configFilePath)
if err != nil {
return fmt.Errorf("open: %s", err)
}

21
cmd/crowdsec-cli/item_metrics.go
View file

@ -18,8 +18,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// XXX: this should not need hub?
func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) error {
func ShowMetrics(hubItem *cwhub.Item) error {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
@ -28,27 +27,15 @@ func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) error {
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, parserName := range hubItem.Parsers {
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, parserName)
parserMetricsTable(color.Output, parserName, metrics)
}
for _, scenarioName := range hubItem.Scenarios {
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, scenarioName)
scenarioMetricsTable(color.Output, scenarioName, metrics)
}
for _, collName := range hubItem.Collections {
subColl := hub.GetItem(cwhub.COLLECTIONS, collName)
if subColl == nil {
return fmt.Errorf("unable to retrieve sub-collection '%s' from '%s'", collName, hubItem.Name)
}
if err := ShowMetrics(hub, subColl); err != nil {
for _, sub := range hubItem.SubItems() {
if err := ShowMetrics(sub); err != nil {
return err
}
}
case cwhub.WAAP_RULES:
log.Fatalf("FIXME: not implemented yet")
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
// no metrics for this item type
}
return nil
}

58
cmd/crowdsec-cli/itemcommands.go
View file

@ -263,21 +263,23 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
}
for _, name := range args {
if hub.GetItem(it.name, name) == nil {
item := hub.GetItem(it.name, name)
if item == nil {
msg := SuggestNearestMessage(hub, it.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
log.Errorf(msg)
continue
}
if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil {
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
@ -346,7 +348,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
removed := 0
for _, item := range items {
didRemove, err := hub.RemoveItem(it.name, item.Name, purge, force)
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
@ -368,27 +370,27 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
}
removed := 0
for _, name := range args {
if !force {
item := hub.GetItem(it.name, name)
if item == nil {
// XXX: this should be in GetItem?
return fmt.Errorf("can't find '%s' in %s", name, it.name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to collections: %s", name, item.BelongsToCollections)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", it.name, name, it.singular)
continue
}
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
didRemove, err := hub.RemoveItem(it.name, name, purge, force)
if !force && len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, item.BelongsToCollections)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", name)
log.Infof("Removed %s", item.Name)
removed++
}
}
@ -451,8 +453,9 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
}
updated := 0
for _, item := range items {
didUpdate, err := hub.UpgradeItem(it.name, item.Name, force)
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
@ -460,7 +463,9 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
updated++
}
}
log.Infof("Updated %d %s", updated, it.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
@ -473,13 +478,20 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
}
updated := 0
for _, name := range args {
didUpdate, err := hub.UpgradeItem(it.name, name, force)
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", name)
log.Infof("Updated %s", item.Name)
updated++
}
}

2
cmd/crowdsec-cli/items.go
View file

@ -157,7 +157,7 @@ func InspectItem(hub *cwhub.Hub, item *cwhub.Item, showMetrics bool) error {
if csConfig.Cscli.Output == "human" && showMetrics {
fmt.Printf("\nCurrent metrics: \n")
if err := ShowMetrics(hub, item); err != nil {
if err := ShowMetrics(item); err != nil {
return err
}
}

4
cmd/crowdsec-cli/main.go
View file

@ -65,10 +65,6 @@ func initConfig() {
log.Debugf("Enabled feature flags: %s", fflist)
}
if csConfig.Cscli == nil {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
}
if flagBranch != "" {
csConfig.Cscli.HubBranch = flagBranch
}

19
cmd/crowdsec-cli/require/branch.go
View file

@ -10,32 +10,33 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func chooseBranch(cfg *csconfig.Config, logger *log.Logger) string {
func chooseBranch(cfg *csconfig.Config) string {
// this was set from config.yaml or flag
if cfg.Cscli.HubBranch != "" {
logger.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
return cfg.Cscli.HubBranch
}
latest, err := cwversion.Latest()
if err != nil {
logger.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
logger.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
logger.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
return "master"
}
if csVersion == "" {
logger.Warning("Crowdsec version is not set, using hub branch 'master'")
log.Warning("Crowdsec version is not set, using hub branch 'master'")
return "master"
}
@ -49,11 +50,7 @@ func chooseBranch(cfg *csconfig.Config, logger *log.Logger) string {
// HubBranch sets the branch (in cscli config) and returns its value
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
func HubBranch(cfg *csconfig.Config) string {
// XXX: we want to be able to suppress logs
// to avoid being too noisy in some commands
logger := log.StandardLogger()
branch := chooseBranch(cfg, logger)
branch := chooseBranch(cfg)
cfg.Cscli.HubBranch = branch

5
cmd/crowdsec-cli/simulation.go
View file

@ -3,11 +3,11 @@ package main
import (
"fmt"
"os"
"slices"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -112,9 +112,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
if err := csConfig.LoadSimulation(); err != nil {
log.Fatal(err)
}
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before using simulation")
}
if csConfig.Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured")
}

15
cmd/crowdsec/crowdsec.go
View file

@ -21,14 +21,9 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/waf"
)
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@ -49,7 +44,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
return csParsers, nil
}
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
inputEventChan = make(chan types.Event)
inputLineChan = make(chan types.Event)
@ -104,7 +99,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
log.Fatalf("starting outputs error : %s", err)
return err
}
@ -136,7 +131,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
return nil
}
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
crowdsecTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveCrowdsec")
go func() {
@ -144,7 +139,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
// this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true
if err := runCrowdsec(cConfig, parsers); err != nil {
if err := runCrowdsec(cConfig, parsers, hub); err != nil {
log.Fatalf("unable to start crowdsec routines: %s", err)
}
}()

2
cmd/crowdsec/main.go
View file

@ -88,7 +88,7 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files))
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, cConfig.ConfigPaths.DataDir, files, &bucketsTomb, buckets, flags.OrderEvent)
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
if err != nil {
return fmt.Errorf("scenario loading failed: %v", err)

8
cmd/crowdsec/output.go
View file

@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
var bucketOverflows []types.Event
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
var err error
ticker := time.NewTicker(1 * time.Second)
@ -70,11 +71,6 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
hub, err := cwhub.GetHub()
if err != nil {
return err
}
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)

19
cmd/crowdsec/serve.go
View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
}
@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
agentReady := make(chan bool, 1)
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
log.Printf("Reload is finished")
@ -342,14 +348,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return fmt.Errorf("crowdsec init: %w", err)
}
// if it's just linting, we're done
if !flags.TestMode {
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
} else {
agentReady <- true

2
pkg/csconfig/cscli.go
View file

@ -21,8 +21,6 @@ func (c *Config) loadCSCLI() error {
c.Cscli = &CscliCfg{}
}
// XXX: HubBranch default should be set here and fed to HubCfg?
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
}

9
pkg/cwhub/cwhub.go
View file

@ -3,3 +3,12 @@
// This includes retrieving the index, the items to install (parsers, scenarios, data files...)
// and managing the dependencies and taints.
package cwhub
import (
"net/http"
"time"
)
var hubClient = &http.Client{
Timeout: 10 * time.Second,
}

6
pkg/cwhub/cwhub_test.go
View file

@ -79,14 +79,14 @@ func envSetup(t *testing.T) *Hub {
setResponseByPath()
log.SetLevel(log.DebugLevel)
defaultTransport := http.DefaultClient.Transport
defaultTransport := hubClient.Transport
t.Cleanup(func() {
http.DefaultClient.Transport = defaultTransport
hubClient.Transport = defaultTransport
})
// Mock the http client
http.DefaultClient.Transport = newMockTransport()
hubClient.Transport = newMockTransport()
hub := testHub(t, true)

49
pkg/cwhub/dataset.go
View file

@ -21,33 +21,23 @@ type DataSet struct {
func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath)
req, err := http.NewRequest(http.MethodGet, url, nil)
resp, err := hubClient.Get(url)
if err != nil {
return err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
return fmt.Errorf("while downloading %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("got HTTP status '%s' from %s", resp.Status, url)
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return err
}
file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644)
file, err := os.Create(destPath)
if err != nil {
return err
}
defer file.Close()
_, err = file.Write(body)
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
}
@ -59,19 +49,6 @@ func downloadFile(url string, destPath string) error {
return nil
}
func GetData(data []types.DataSource, dataDir string) error {
for _, dataS := range data {
destPath := filepath.Join(dataDir, dataS.DestPath)
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
if err := downloadFile(dataS.SourceURL, destPath); err != nil {
return err
}
}
return nil
}
// downloadData downloads the data files for an item
func downloadData(dataFolder string, force bool, reader io.Reader) error {
dec := yaml.NewDecoder(reader)
@ -87,17 +64,15 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error {
return fmt.Errorf("while reading file: %w", err)
}
download := false
for _, dataS := range data.Data {
if _, err := os.Stat(filepath.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
download = true
}
}
destPath := filepath.Join(dataFolder, dataS.DestPath)
if download || force {
if err := GetData(data.Data, dataFolder); err != nil {
return fmt.Errorf("while getting data: %w", err)
if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
if err := downloadFile(dataS.SourceURL, destPath); err != nil {
return fmt.Errorf("while getting data: %w", err)
}
}
}
}

123
pkg/cwhub/enable.go
View file

@ -10,25 +10,24 @@ import (
log "github.com/sirupsen/logrus"
)
// EnableItem creates a symlink between actual config file at hub.HubDir and hub.ConfigDir
// enable creates a symlink between actual config file at hub.HubDir and hub.ConfigDir
// Handles collections recursively
// XXX: called from config_restore otherwise no need to export
func (h *Hub) EnableItem(target *Item) error {
parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
func (i *Item) enable() error {
parentDir := filepath.Clean(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/")
// create directories if needed
if target.Installed {
if target.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
if i.Installed {
if i.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
}
if target.IsLocal() {
return fmt.Errorf("%s is local, won't enable", target.Name)
if i.IsLocal() {
return fmt.Errorf("%s is local, won't enable", i.Name)
}
// if it's a collection, check sub-items even if the collection file itself is up-to-date
if target.UpToDate && target.Type != COLLECTIONS {
log.Tracef("%s is installed and up-to-date, skip.", target.Name)
if i.UpToDate && !i.HasSubItems() {
log.Tracef("%s is installed and up-to-date, skip.", i.Name)
return nil
}
}
@ -41,31 +40,26 @@ func (h *Hub) EnableItem(target *Item) error {
}
}
// install sub-items if it's a collection
for _, sub := range target.SubItems() {
val, ok := h.Items[sub.Type][sub.Name]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name)
}
if err := h.EnableItem(&val); err != nil {
// install sub-items if any
for _, sub := range i.SubItems() {
if err := sub.enable(); err != nil {
return fmt.Errorf("while installing %s: %w", sub.Name, err)
}
}
// check if file already exists where it should in configdir (eg /etc/crowdsec/collections/)
if _, err := os.Lstat(parentDir + "/" + target.FileName); !os.IsNotExist(err) {
log.Infof("%s already exists.", parentDir+"/"+target.FileName)
if _, err := os.Lstat(parentDir + "/" + i.FileName); !os.IsNotExist(err) {
log.Infof("%s already exists.", parentDir+"/"+i.FileName)
return nil
}
// hub.ConfigDir + target.RemotePath
srcPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath)
srcPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
if err != nil {
return fmt.Errorf("while getting source path: %w", err)
}
dstPath, err := filepath.Abs(parentDir + "/" + target.FileName)
dstPath, err := filepath.Abs(parentDir + "/" + i.FileName)
if err != nil {
return fmt.Errorf("while getting destination path: %w", err)
}
@ -74,38 +68,36 @@ func (h *Hub) EnableItem(target *Item) error {
return fmt.Errorf("while creating symlink from %s to %s: %w", srcPath, dstPath, err)
}
log.Infof("Enabled %s: %s", target.Type, target.Name)
target.Installed = true
h.Items[target.Type][target.Name] = *target
log.Infof("Enabled %s: %s", i.Type, i.Name)
i.Installed = true
return nil
}
func (h *Hub) purgeItem(target Item) (Item, error) {
itempath := h.local.HubDir + "/" + target.RemotePath
// purge removes the actual config file that was downloaded
func (i *Item) purge() error {
itempath := i.hub.local.HubDir + "/" + i.RemotePath
// disable hub file
if err := os.Remove(itempath); err != nil {
return target, fmt.Errorf("while removing file: %w", err)
return fmt.Errorf("while removing file: %w", err)
}
target.Downloaded = false
log.Infof("Removed source file [%s]: %s", target.Name, itempath)
h.Items[target.Type][target.Name] = target
i.Downloaded = false
log.Infof("Removed source file [%s]: %s", i.Name, itempath)
return target, nil
return nil
}
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
// disable removes the symlink to the downloaded content, also removes the content if purge is true
func (i *Item) disable(purge bool, force bool) error {
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
var err error
// already disabled, noop unless purge
if !target.Installed {
if !i.Installed {
if purge {
*target, err = h.purgeItem(*target)
if err != nil {
if err = i.purge(); err != nil {
return err
}
}
@ -113,43 +105,36 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
return nil
}
if target.IsLocal() {
return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
if i.IsLocal() {
return fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
}
if target.Tainted && !force {
return fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
if i.Tainted && !force {
return fmt.Errorf("%s is tainted, use '--force' to overwrite", i.Name)
}
// disable sub-items if any - it's a collection
for _, sub := range target.SubItems() {
// XXX: we do this already when syncing, do we really need to do consistency checks here and there?
val, ok := h.Items[sub.Type][sub.Name]
if !ok {
log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name)
continue
}
for _, sub := range i.SubItems() {
// check if the item doesn't belong to another collection before removing it
toRemove := true
removeSub := true
for _, collection := range val.BelongsToCollections {
if collection != target.Name {
toRemove = false
for _, collection := range sub.BelongsToCollections {
if collection != i.Name {
removeSub = false
break
}
}
if toRemove {
if err = h.DisableItem(&val, purge, force); err != nil {
if removeSub {
if err = sub.disable(purge, force); err != nil {
return fmt.Errorf("while disabling %s: %w", sub.Name, err)
}
} else {
log.Infof("%s was not removed because it belongs to another collection", val.Name)
log.Infof("%s was not removed because it belongs to another collection", sub.Name)
}
}
syml, err := filepath.Abs(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
syml, err := filepath.Abs(i.hub.local.InstallDir + "/" + i.Type + "/" + i.Stage + "/" + i.FileName)
if err != nil {
return err
}
@ -158,13 +143,13 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
if os.IsNotExist(err) {
// we only accept to "delete" non existing items if it's a forced purge
if !purge && !force {
return fmt.Errorf("can't delete %s: %s doesn't exist", target.Name, syml)
return fmt.Errorf("can't delete %s: %s doesn't exist", i.Name, syml)
}
} else {
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml)
return fmt.Errorf("%s isn't managed by hub", target.Name)
log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
hubpath, err := os.Readlink(syml)
@ -172,34 +157,30 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
return fmt.Errorf("while reading symlink: %w", err)
}
absPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath)
absPath, err := filepath.Abs(i.hub.local.HubDir + "/" + i.RemotePath)
if err != nil {
return fmt.Errorf("while abs path: %w", err)
}
if hubpath != absPath {
log.Warningf("%s (%s) isn't a symlink to %s", target.Name, syml, absPath)
return fmt.Errorf("%s isn't managed by hub", target.Name)
log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, absPath)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
// remove the symlink
if err = os.Remove(syml); err != nil {
return fmt.Errorf("while removing symlink: %w", err)
}
log.Infof("Removed symlink [%s]: %s", target.Name, syml)
log.Infof("Removed symlink [%s]: %s", i.Name, syml)
}
target.Installed = false
i.Installed = false
if purge {
*target, err = h.purgeItem(*target)
if err != nil {
if err = i.purge(); err != nil {
return err
}
}
h.Items[target.Type][target.Name] = *target
return nil
}

45
pkg/cwhub/enable_test.go
View file

@ -8,84 +8,81 @@ import (
"github.com/stretchr/testify/require"
)
func testInstall(hub *Hub, t *testing.T, item Item) {
func testInstall(hub *Hub, t *testing.T, item *Item) {
// Install the parser
err := hub.DownloadLatest(&item, false, false)
err := item.downloadLatest(false, false)
require.NoError(t, err, "failed to download %s", item.Name)
_, err = hub.LocalSync()
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
err = hub.EnableItem(&item)
err = item.enable()
require.NoError(t, err, "failed to enable %s", item.Name)
_, err = hub.LocalSync()
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
}
func testTaint(hub *Hub, t *testing.T, item Item) {
func testTaint(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
defer f.Close()
_, err = f.WriteString("tainted")
require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
// truncate the file
f, err := os.Create(item.LocalPath)
require.NoError(t, err)
f.Close()
// Local sync and check status
_, err = hub.LocalSync()
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
}
func testUpdate(hub *Hub, t *testing.T, item Item) {
func testUpdate(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
err := hub.DownloadLatest(&item, true, true)
err := item.downloadLatest(true, true)
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status
_, err = hub.LocalSync()
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisable(hub *Hub, t *testing.T, item Item) {
func testDisable(hub *Hub, t *testing.T, item *Item) {
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
// Remove
err := hub.DisableItem(&item, false, false)
err := item.disable(false, false)
require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status
warns, err := hub.LocalSync()
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings: %+v", warns)
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
// Purge
err = hub.DisableItem(&item, true, false)
err = item.disable(true, false)
require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status
warns, err = hub.LocalSync()
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings: %+v", warns)
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)

241
pkg/cwhub/helpers.go
View file

@ -19,15 +19,10 @@ import (
log "github.com/sirupsen/logrus"
)
// InstallItem installs an item from the hub
func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error {
item := h.GetItem(itemType, name)
if item == nil {
return fmt.Errorf("unable to retrieve item: %s", name)
}
if downloadOnly && item.Downloaded && item.UpToDate {
log.Warningf("%s is already downloaded and up-to-date", item.Name)
// Install installs the item from the hub, downloading it if needed
func (i *Item) Install(force bool, downloadOnly bool) error {
if downloadOnly && i.Downloaded && i.UpToDate {
log.Warningf("%s is already downloaded and up-to-date", i.Name)
if !force {
return nil
@ -35,90 +30,68 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
}
// XXX: confusing semantic between force and updateOnly?
if err := h.DownloadLatest(item, force, true); err != nil {
return fmt.Errorf("while downloading %s: %w", item.Name, err)
}
if err := h.AddItem(*item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
if err := i.downloadLatest(force, true); err != nil {
return fmt.Errorf("while downloading %s: %w", i.Name, err)
}
if downloadOnly {
// XXX: should get the path from DownloadLatest
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.local.HubDir, item.RemotePath))
// XXX: should get the path from downloadLatest
log.Infof("Downloaded %s to %s", i.Name, filepath.Join(i.hub.local.HubDir, i.RemotePath))
return nil
}
// XXX: should we stop here if the item is already installed?
if err := h.EnableItem(item); err != nil {
return fmt.Errorf("while enabling %s: %w", item.Name, err)
if err := i.enable(); err != nil {
return fmt.Errorf("while enabling %s: %w", i.Name, err)
}
if err := h.AddItem(*item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
}
log.Infof("Enabled %s", item.Name)
log.Infof("Enabled %s", i.Name)
return nil
}
// RemoveItem disables one item, optionally removing the downloaded content
func (h *Hub) RemoveItem(itemType string, name string, purge bool, forceAction bool) (bool, error) {
// Remove disables the item, optionally removing the downloaded content
func (i *Item) Remove(purge bool, forceAction bool) (bool, error) {
removed := false
item := h.GetItem(itemType, name)
if item == nil {
return false, fmt.Errorf("can't find '%s' in %s", name, itemType)
}
if !item.Downloaded {
log.Infof("removing %s: not downloaded -- no removal required", item.Name)
if !i.Downloaded {
log.Infof("removing %s: not downloaded -- no removal required", i.Name)
return false, nil
}
if !item.Installed && !purge {
log.Infof("removing %s: already uninstalled", item.Name)
if !i.Installed && !purge {
log.Infof("removing %s: already uninstalled", i.Name)
return false, nil
}
if err := h.DisableItem(item, purge, forceAction); err != nil {
return false, fmt.Errorf("unable to disable %s: %w", item.Name, err)
if err := i.disable(purge, forceAction); err != nil {
return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
}
// XXX: should take the value from DisableItem
// XXX: should take the value from disable()
removed = true
if err := h.AddItem(*item); err != nil {
return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err)
}
return removed, nil
}
// UpgradeItem upgrades an item from the hub
func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error) {
// Upgrade downloads and applies the last version from the hub
func (i *Item) Upgrade(force bool) (bool, error) {
updated := false
item := h.GetItem(itemType, name)
if item == nil {
return false, fmt.Errorf("can't find '%s' in %s", name, itemType)
if !i.Downloaded {
return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
}
if !item.Downloaded {
return false, fmt.Errorf("can't upgrade %s: not installed", item.Name)
if !i.Installed {
return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", i.Name)
}
if !item.Installed {
return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", item.Name)
}
if i.UpToDate {
log.Infof("%s: up-to-date", i.Name)
if item.UpToDate {
log.Infof("%s: up-to-date", item.Name)
if err := h.DownloadDataIfNeeded(*item, force); err != nil {
return false, fmt.Errorf("%s: download failed: %w", item.Name, err)
if err := i.DownloadDataIfNeeded(force); err != nil {
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
}
if !force {
@ -127,160 +100,139 @@ func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error
}
}
if err := h.DownloadLatest(item, force, true); err != nil {
return false, fmt.Errorf("%s: download failed: %w", item.Name, err)
if err := i.downloadLatest(force, true); err != nil {
return false, fmt.Errorf("%s: download failed: %w", i.Name, err)
}
if !item.UpToDate {
if item.Tainted {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, item.Name)
} else if item.IsLocal() {
log.Infof("%v %s is local", emoji.Prohibited, item.Name)
if !i.UpToDate {
if i.Tainted {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
} else if i.IsLocal() {
log.Infof("%v %s is local", emoji.Prohibited, i.Name)
}
} else {
// a check on stdout is used while scripting to know if the hub has been upgraded
// and a configuration reload is required
// TODO: use a better way to communicate this
fmt.Printf("updated %s\n", item.Name)
log.Infof("%v %s: updated", emoji.Package, item.Name)
fmt.Printf("updated %s\n", i.Name)
log.Infof("%v %s: updated", emoji.Package, i.Name)
updated = true
}
if err := h.AddItem(*item); err != nil {
return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err)
}
return updated, nil
}
// DownloadLatest will download the latest version of Item to the tdir directory
func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error {
// XXX: should return the path of the downloaded file (taken from DownloadItem)
log.Debugf("Downloading %s %s", target.Type, target.Name)
// downloadLatest will download the latest version of Item to the tdir directory
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error {
// XXX: should return the path of the downloaded file (taken from download())
log.Debugf("Downloading %s %s", i.Type, i.Name)
if target.Type != COLLECTIONS {
if !target.Installed && updateOnly && target.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", target.Name)
return nil
}
return h.DownloadItem(target, overwrite)
}
// collection
for _, sub := range target.SubItems() {
val, ok := h.Items[sub.Type][sub.Name]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", sub.Type, sub.Name, target.Name)
}
if !val.Installed && updateOnly && val.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", target.Name)
for _, sub := range i.SubItems() {
if !sub.Installed && updateOnly && sub.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name)
continue
}
log.Debugf("Download %s sub-item: %s %s (%t -> %t)", target.Name, sub.Type, sub.Name, target.Installed, updateOnly)
log.Debugf("Download %s sub-item: %s %s (%t -> %t)", i.Name, sub.Type, sub.Name, i.Installed, updateOnly)
// recurse as it's a collection
if sub.Type == COLLECTIONS {
if sub.HasSubItems() {
log.Tracef("collection, recurse")
if err := h.DownloadLatest(&val, overwrite, updateOnly); err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
if err := sub.downloadLatest(overwrite, updateOnly); err != nil {
return fmt.Errorf("while downloading %s: %w", sub.Name, err)
}
}
downloaded := val.Downloaded
downloaded := sub.Downloaded
if err := h.DownloadItem(&val, overwrite); err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
if err := sub.download(overwrite); err != nil {
return fmt.Errorf("while downloading %s: %w", sub.Name, err)
}
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !val.Installed && !downloaded {
if err := h.EnableItem(&val); err != nil {
return fmt.Errorf("enabling '%s': %w", val.Name, err)
// We check if sub.Downloaded is false because maybe the item has been disabled by the user.
if !sub.Installed && !downloaded {
if err := sub.enable(); err != nil {
return fmt.Errorf("enabling '%s': %w", sub.Name, err)
}
}
h.Items[sub.Type][sub.Name] = val
}
if err := h.DownloadItem(target, overwrite); err != nil {
if !i.Installed && updateOnly && i.Downloaded {
log.Debugf("skipping upgrade of %s: not installed", i.Name)
return nil
}
if err := i.download(overwrite); err != nil {
return fmt.Errorf("failed to download item: %w", err)
}
return nil
}
func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
url, err := h.remote.urlTo(target.RemotePath)
func (i *Item) download(overwrite bool) error {
url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
return fmt.Errorf("failed to build hub item request: %w", err)
}
tdir := h.local.HubDir
tdir := i.hub.local.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
if target.Tainted {
log.Debugf("%s: tainted, not updated", target.Name)
if i.Tainted {
log.Debugf("%s: tainted, not updated", i.Name)
return nil
}
if target.UpToDate {
if i.UpToDate {
// We still have to check if data files are present
log.Debugf("%s: up-to-date, not updated", target.Name)
log.Debugf("%s: up-to-date, not updated", i.Name)
}
}
req, err := http.NewRequest(http.MethodGet, url, nil)
resp, err := hubClient.Get(url)
if err != nil {
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
return fmt.Errorf("while downloading %s: %w", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
return fmt.Errorf("while downloading %s: %w", url, err)
}
hash := sha256.New()
if _, err = hash.Write(body); err != nil {
return fmt.Errorf("while hashing %s: %w", target.Name, err)
return fmt.Errorf("while hashing %s: %w", i.Name, err)
}
meow := hex.EncodeToString(hash.Sum(nil))
if meow != target.Versions[target.Version].Digest {
if meow != i.Versions[i.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
return fmt.Errorf("invalid download hash for %s", target.Name)
return fmt.Errorf("invalid download hash for %s", i.Name)
}
//all good, install
//check if parent dir exists
tmpdirs := strings.Split(tdir+"/"+target.RemotePath, "/")
tmpdirs := strings.Split(tdir+"/"+i.RemotePath, "/")
parentDir := strings.Join(tmpdirs[:len(tmpdirs)-1], "/")
// ensure that target file is within target dir
finalPath, err := filepath.Abs(tdir + "/" + target.RemotePath)
finalPath, err := filepath.Abs(tdir + "/" + i.RemotePath)
if err != nil {
return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+target.RemotePath, err)
return fmt.Errorf("filepath.Abs error on %s: %w", tdir+"/"+i.RemotePath, err)
}
if !strings.HasPrefix(finalPath, tdir) {
return fmt.Errorf("path %s escapes %s, abort", target.RemotePath, tdir)
return fmt.Errorf("path %s escapes %s, abort", i.RemotePath, tdir)
}
// check dir
@ -294,17 +246,16 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
// check actual file
if _, err = os.Stat(finalPath); !os.IsNotExist(err) {
log.Warningf("%s: overwrite", target.Name)
log.Debugf("target: %s/%s", tdir, target.RemotePath)
log.Warningf("%s: overwrite", i.Name)
log.Debugf("target: %s/%s", tdir, i.RemotePath)
} else {
log.Infof("%s: OK", target.Name)
log.Infof("%s: OK", i.Name)
}
f, err := os.OpenFile(tdir+"/"+target.RemotePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
f, err := os.Create(tdir + "/" + i.RemotePath)
if err != nil {
return fmt.Errorf("while opening file: %w", err)
}
defer f.Close()
_, err = f.Write(body)
@ -312,22 +263,20 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
return fmt.Errorf("while writing file: %w", err)
}
target.Downloaded = true
target.Tainted = false
target.UpToDate = true
i.Downloaded = true
i.Tainted = false
i.UpToDate = true
if err = downloadData(h.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
if err = downloadData(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
}
h.Items[target.Type][target.Name] = *target
return nil
}
// DownloadDataIfNeeded downloads the data files for an item
func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.local.InstallDir, target.Type, target.Stage, target.FileName)
// DownloadDataIfNeeded downloads the data files for the item
func (i *Item) DownloadDataIfNeeded(force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", i.hub.local.InstallDir, i.Type, i.Stage, i.FileName)
itemFile, err := os.Open(itemFilePath)
if err != nil {
@ -336,7 +285,7 @@ func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
defer itemFile.Close()
if err = downloadData(h.local.InstallDataDir, force, itemFile); err != nil {
if err = downloadData(i.hub.local.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}

42
pkg/cwhub/helpers_test.go
View file

@ -17,7 +17,8 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -25,10 +26,9 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
// This is the scenario that gets added in next version of collection
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
pushUpdateToCollectionInHub()
@ -49,10 +49,11 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.True(t, didUpdate)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
@ -68,16 +69,18 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false)
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
didRemove, err := item.Remove(false, false)
require.NoError(t, err)
require.True(t, didRemove)
@ -98,7 +101,8 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
hub, err = NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err)
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.False(t, didUpdate)
@ -125,16 +129,18 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false)
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
didRemove, err := item.Remove(false, false)
require.NoError(t, err)
require.True(t, didRemove)
@ -164,7 +170,8 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
hub = getHubOrFail(t, hub.local, remote)
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.True(t, didUpdate)
@ -173,14 +180,11 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
}
func assertCollectionDepsInstalled(t *testing.T, collection string) {
func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
t.Helper()
hub, err := GetHub()
require.NoError(t, err)
c := hub.Items[COLLECTIONS][collection]
require.NoError(t, hub.CollectDepsCheck(&c))
require.NoError(t, hub.checkSubItems(c))
}
func pushUpdateToCollectionInHub() {

32
pkg/cwhub/hub.go
View file

@ -17,19 +17,7 @@ type Hub struct {
remote *RemoteHubCfg
skippedLocal int
skippedTainted int
}
var theHub *Hub
// GetHub returns the hub singleton
// it returns an error if it's not initialized to avoid nil dereference
// XXX: convenience function that we should get rid of at some point
func GetHub() (*Hub, error) {
if theHub == nil {
return nil, fmt.Errorf("hub not initialized")
}
return theHub, nil
Warnings []string
}
func (h *Hub) GetDataDir() string {
@ -51,20 +39,20 @@ func NewHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg, downloadIndex boo
log.Debugf("loading hub idx %s", local.HubIndexFile)
theHub = &Hub{
hub := &Hub{
local: local,
remote: remote,
}
if err := theHub.parseIndex(); err != nil {
if err := hub.parseIndex(); err != nil {
return nil, fmt.Errorf("failed to load index: %w", err)
}
if _, err := theHub.LocalSync(); err != nil {
if err := hub.localSync(); err != nil {
return nil, fmt.Errorf("failed to sync items: %w", err)
}
return theHub, nil
return hub, nil
}
// parseIndex takes the content of an index file and fills the map of associated parsers/scenarios/collections
@ -85,6 +73,7 @@ func (h *Hub) parseIndex() error {
log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
for name, item := range h.Items[itemType] {
item.hub = h
item.Name = name
// if the item has no (redundant) author, take it from the json key
@ -95,15 +84,8 @@ func (h *Hub) parseIndex() error {
item.Type = itemType
x := strings.Split(item.RemotePath, "/")
item.FileName = x[len(x)-1]
h.Items[itemType][name] = item
// if it's a collection, check its sub-items are present
// XXX should be done later, maybe report all missing at once?
for _, sub := range item.SubItems() {
if _, ok := h.Items[sub.Type][sub.Name]; !ok {
log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, item.Name)
}
}
item.logMissingSubItems()
}
}

3
pkg/cwhub/hub_test.go
View file

@ -21,9 +21,6 @@ func TestInitHubUpdate(t *testing.T) {
_, err := NewHub(hub.local, remote, true)
require.NoError(t, err)
_, err = GetHub()
require.NoError(t, err)
}
func TestDownloadIndex(t *testing.T) {

154
pkg/cwhub/items.go
View file

@ -6,6 +6,7 @@ import (
"github.com/Masterminds/semver/v3"
"github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
)
const (
@ -28,18 +29,21 @@ const (
// The order is important, as it is used to range over sub-items in collections
var ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, WAAP_CONFIGS, WAAP_RULES, COLLECTIONS}
type HubItems map[string]map[string]Item
type HubItems map[string]map[string]*Item
// ItemVersion is used to detect the version of a given item
// by comparing the hash of each version to the local file.
// If the item does not match any known version, it is considered tainted.
type ItemVersion struct {
Digest string `json:"digest,omitempty"` // meow
Deprecated bool `json:"deprecated,omitempty"` // XXX: do we keep this?
Digest string `json:"digest,omitempty"` // meow
Deprecated bool `json:"deprecated,omitempty"`
}
// Item represents an object managed in the hub. It can be a parser, scenario, collection..
type Item struct {
// back pointer to the hub, to retrieve subitems and call install/remove methods
hub *Hub
// descriptive info
Type string `json:"type,omitempty" yaml:"type,omitempty"` // can be any of the ItemTypes
Stage string `json:"stage,omitempty" yaml:"stage,omitempty"` // Stage for parser|postoverflow: s00-raw/s01-...
@ -59,10 +63,10 @@ type Item struct {
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` // the local meow
Installed bool `json:"installed,omitempty"` // XXX: should we remove omitempty from bool fields?
Downloaded bool `json:"downloaded,omitempty"`
UpToDate bool `json:"up_to_date,omitempty"`
Tainted bool `json:"tainted,omitempty"` // has it been locally modified?
Installed bool `json:"installed"`
Downloaded bool `json:"downloaded"`
UpToDate bool `json:"up_to_date"`
Tainted bool `json:"tainted"` // has it been locally modified?
// if it's a collection, it can have sub items
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
@ -73,9 +77,8 @@ type Item struct {
WaapRules []string `json:"waap_rules,omitempty" yaml:"waap_rules,omitempty"`
}
type SubItem struct {
Type string
Name string
func (i *Item) HasSubItems() bool {
return i.Type == COLLECTIONS
}
func (i *Item) IsLocal() bool {
@ -90,7 +93,7 @@ func (i Item) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
Alias
Local bool `json:"local"` // XXX: omitempty?
Local bool `json:"local"`
}{
Alias: Alias(i),
Local: i.IsLocal(),
@ -112,51 +115,97 @@ func (i Item) MarshalYAML() (interface{}, error) {
}, nil
}
// SubItems returns the list of sub items for a given item (typically a collection)
func (i *Item) SubItems() []SubItem {
sub := make([]SubItem,
len(i.Parsers)+
len(i.PostOverflows)+
len(i.Scenarios)+
len(i.Collections)+
len(i.WaapConfigs)+
len(i.WaapRules))
n := 0
// SubItems returns a slice of sub-item pointers, excluding the ones that were not found
func (i *Item) SubItems() []*Item {
sub := make([]*Item, 0)
for _, name := range i.Parsers {
sub[n] = SubItem{Type: PARSERS, Name: name}
n++
s := i.hub.GetItem(PARSERS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.PostOverflows {
sub[n] = SubItem{Type: POSTOVERFLOWS, Name: name}
n++
s := i.hub.GetItem(POSTOVERFLOWS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.Scenarios {
sub[n] = SubItem{Type: SCENARIOS, Name: name}
n++
s := i.hub.GetItem(SCENARIOS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.WaapConfigs {
sub[n] = SubItem{Type: WAAP_CONFIGS, Name: name}
n++
s := i.hub.GetItem(WAAP_CONFIGS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.WaapRules {
sub[n] = SubItem{Type: WAAP_RULES, Name: name}
n++
s := i.hub.GetItem(WAAP_RULES, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.Collections {
sub[n] = SubItem{Type: COLLECTIONS, Name: name}
n++
s := i.hub.GetItem(COLLECTIONS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
return sub
}
func (i *Item) logMissingSubItems() {
if !i.HasSubItems() {
return
}
for _, subName := range i.Parsers {
if i.hub.GetItem(PARSERS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, PARSERS, i.Name)
}
}
for _, subName := range i.Scenarios {
if i.hub.GetItem(SCENARIOS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, SCENARIOS, i.Name)
}
}
for _, subName := range i.PostOverflows {
if i.hub.GetItem(POSTOVERFLOWS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, POSTOVERFLOWS, i.Name)
}
}
for _, subName := range i.Collections {
if i.hub.GetItem(COLLECTIONS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, COLLECTIONS, i.Name)
}
}
}
// Status returns the status of the item as a string and an emoji
// ie. "enabled,update-available" and emoji.Warning
func (i *Item) Status() (string, emoji.Emoji) {
@ -228,23 +277,13 @@ func (i *Item) validPath(dirName, fileName string) bool {
}
// GetItemMap returns the map of items for a given type
func (h *Hub) GetItemMap(itemType string) map[string]Item {
m, ok := h.Items[itemType]
if !ok {
return nil
}
return m
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
return h.Items[itemType]
}
// GetItem returns the item from hub based on its type and full name (author/name)
func (h *Hub) GetItem(itemType string, itemName string) *Item {
m, ok := h.GetItemMap(itemType)[itemName]
if !ok {
return nil
}
return &m
return h.GetItemMap(itemType)[itemName]
}
// GetItemNames returns the list of item (full) names for a given type
@ -264,27 +303,14 @@ func (h *Hub) GetItemNames(itemType string) []string {
return names
}
// AddItem adds an item to the hub index
func (h *Hub) AddItem(item Item) error {
for _, t := range ItemTypes {
if t == item.Type {
h.Items[t][item.Name] = item
return nil
}
}
// XXX: can this happen?
return fmt.Errorf("ItemType %s is unknown", item.Type)
}
// GetInstalledItems returns the list of installed items
func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) {
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
retItems := make([]Item, 0)
retItems := make([]*Item, 0)
for _, item := range items {
if item.Installed {
@ -304,8 +330,8 @@ func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
retStr := make([]string, len(items))
for i, it := range items {
retStr[i] = it.Name
for idx, it := range items {
retStr[idx] = it.Name
}
return retStr, nil

9
pkg/cwhub/items_test.go
View file

@ -4,8 +4,6 @@ import (
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/go-cs-lib/cstest"
)
func TestItemStatus(t *testing.T) {
@ -62,14 +60,9 @@ func TestGetters(t *testing.T) {
// Add item and get it
item.Name += "nope"
err := hub.AddItem(*item)
require.NoError(t, err)
hub.Items[item.Type][item.Name] = item
newitem := hub.GetItem(COLLECTIONS, item.Name)
require.NotNil(t, newitem)
item.Type = "ratata"
err = hub.AddItem(*item)
cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
}
}

15
pkg/cwhub/remote.go
View file

@ -22,6 +22,7 @@ func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
return "", ErrNilRemoteHub
}
// the template must contain two string placeholders
if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
}
@ -40,12 +41,7 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error {
return fmt.Errorf("failed to build hub index request: %w", err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return fmt.Errorf("failed to build request for hub index: %w", err)
}
resp, err := http.DefaultClient.Do(req)
resp, err := hubClient.Get(url)
if err != nil {
return fmt.Errorf("failed http request for hub index: %w", err)
}
@ -53,10 +49,10 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error {
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
return IndexNotFoundError{req.URL.String(), r.Branch}
return IndexNotFoundError{url, r.Branch}
}
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, req.URL.String())
return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
}
body, err := io.ReadAll(resp.Body)
@ -74,8 +70,7 @@ func (r *RemoteHubCfg) downloadIndex(localPath string) error {
return nil
}
file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
file, err := os.Create(localPath)
if err != nil {
return fmt.Errorf("while opening hub index file: %w", err)
}

82
pkg/cwhub/sync.go
View file

@ -135,20 +135,21 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
// sortedVersions returns the input data, sorted in reverse order by semver
func sortedVersions(raw []string) ([]string, error) {
vs := make([]*semver.Version, len(raw))
for i, r := range raw {
for idx, r := range raw {
v, err := semver.NewVersion(r)
if err != nil {
return nil, fmt.Errorf("%s: %w", r, err)
}
vs[i] = v
vs[idx] = v
}
sort.Sort(sort.Reverse(semver.Collection(vs)))
ret := make([]string, len(vs))
for i, v := range vs {
ret[i] = v.Original()
for idx, v := range vs {
ret[idx] = v.Original()
}
return ret, nil
@ -186,7 +187,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
}
/*
we can encounter 'collections' in the form of a symlink :
we can encounter 'collections' in the form of a symlink:
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
when the collection is installed, both files are created
*/
@ -212,11 +213,11 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
if local && !inhub {
log.Tracef("%s is a local file, skip", path)
h.skippedLocal++
// log.Infof("local scenario, skip.")
_, fileName := filepath.Split(path)
h.Items[info.ftype][info.fname] = Item{
h.Items[info.ftype][info.fname] = &Item{
hub: h,
Name: info.fname,
Stage: info.stage,
Installed: true,
@ -341,74 +342,61 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
return nil
}
func (h *Hub) CollectDepsCheck(v *Item) error {
if v.Type != COLLECTIONS {
// checkSubItems checks for the presence, taint and version state of sub-items
func (h *Hub) checkSubItems(v *Item) error {
if !v.HasSubItems() {
return nil
}
if v.versionStatus() != VersionUpToDate { // not up-to-date
if v.versionStatus() != VersionUpToDate {
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
return nil
}
// if it's a collection, ensure all the items are installed, or tag it as tainted
// ensure all the sub-items are installed, or tag the parent as tainted
log.Tracef("checking submembers of %s installed:%t", v.Name, v.Installed)
for _, sub := range v.SubItems() {
subItem, ok := h.Items[sub.Type][sub.Name]
if !ok {
return fmt.Errorf("referred %s %s in collection %s doesn't exist", sub.Type, sub.Name, v.Name)
}
log.Tracef("check %s installed:%t", subItem.Name, subItem.Installed)
log.Tracef("check %s installed:%t", sub.Name, sub.Installed)
if !v.Installed {
continue
}
if subItem.Type == COLLECTIONS {
log.Tracef("collec, recurse.")
if err := h.CollectDepsCheck(&subItem); err != nil {
if subItem.Tainted {
v.Tainted = true
}
return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
if err := h.checkSubItems(sub); err != nil {
if sub.Tainted {
v.Tainted = true
}
h.Items[sub.Type][sub.Name] = subItem
return fmt.Errorf("sub collection %s is broken: %w", sub.Name, err)
}
// propagate the state of sub-items to set
if subItem.Tainted {
if sub.Tainted {
v.Tainted = true
return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name)
}
if !subItem.Installed && v.Installed {
if !sub.Installed && v.Installed {
v.Tainted = true
return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name)
}
if !subItem.UpToDate {
if !sub.UpToDate {
v.UpToDate = false
return fmt.Errorf("outdated %s %s", sub.Type, sub.Name)
}
if !slices.Contains(subItem.BelongsToCollections, v.Name) {
subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
if !slices.Contains(sub.BelongsToCollections, v.Name) {
sub.BelongsToCollections = append(sub.BelongsToCollections, v.Name)
}
h.Items[sub.Type][sub.Name] = subItem
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, v.Tainted, v.UpToDate)
}
return nil
}
func (h *Hub) SyncDir(dir string) ([]string, error) {
func (h *Hub) syncDir(dir string) ([]string, error) {
warnings := []string{}
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
@ -429,7 +417,7 @@ func (h *Hub) SyncDir(dir string) ([]string, error) {
}
}
for name, item := range h.Items[COLLECTIONS] {
for _, item := range h.Items[COLLECTIONS] {
if !item.Installed {
continue
}
@ -437,9 +425,8 @@ func (h *Hub) SyncDir(dir string) ([]string, error) {
vs := item.versionStatus()
switch vs {
case VersionUpToDate: // latest
if err := h.CollectDepsCheck(&item); err != nil {
if err := h.checkSubItems(item); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
h.Items[COLLECTIONS][name] = item
}
case VersionUpdateAvailable: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
@ -456,18 +443,23 @@ func (h *Hub) SyncDir(dir string) ([]string, error) {
}
// Updates the info from HubInit() with the local state
func (h *Hub) LocalSync() ([]string, error) {
func (h *Hub) localSync() error {
h.skippedLocal = 0
h.skippedTainted = 0
h.Warnings = []string{}
warnings, err := h.SyncDir(h.local.InstallDir)
warnings, err := h.syncDir(h.local.InstallDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
}
if _, err = h.SyncDir(h.local.HubDir); err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
h.Warnings = append(h.Warnings, warnings...)
if warnings, err = h.syncDir(h.local.HubDir); err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
}
return warnings, nil
h.Warnings = append(h.Warnings, warnings...)
return nil
}

6
pkg/hubtest/hubtest_item.go
View file

@ -410,7 +410,7 @@ func (t *HubTestItem) InstallHub() error {
ret := hub.GetItemMap(cwhub.PARSERS)
for parserName, item := range ret {
if item.Installed {
if err := hub.DownloadDataIfNeeded(item, true); err != nil {
if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
}
@ -422,7 +422,7 @@ func (t *HubTestItem) InstallHub() error {
ret = hub.GetItemMap(cwhub.SCENARIOS)
for scenarioName, item := range ret {
if item.Installed {
if err := hub.DownloadDataIfNeeded(item, true); err != nil {
if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
}
@ -434,7 +434,7 @@ func (t *HubTestItem) InstallHub() error {
ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
for postoverflowName, item := range ret {
if item.Installed {
if err := hub.DownloadDataIfNeeded(item, true); err != nil {
if err := item.DownloadDataIfNeeded(true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
}

11
pkg/leakybucket/buckets_test.go
View file

@ -41,9 +41,10 @@ func TestBucket(t *testing.T) {
hubCfg := &csconfig.LocalHubCfg{
HubDir: filepath.Join(testdata, "hub"),
HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
InstallDataDir: testdata,
}
_, err := cwhub.NewHub(hubCfg, nil, false)
hub, err := cwhub.NewHub(hubCfg, nil, false)
if err != nil {
t.Fatalf("failed to init hub: %s", err)
}
@ -54,7 +55,7 @@ func TestBucket(t *testing.T) {
}
if envSetting != "" {
if err := testOneBucket(t, envSetting, tomb); err != nil {
if err := testOneBucket(t, hub, envSetting, tomb); err != nil {
t.Fatalf("Test '%s' failed : %s", envSetting, err)
}
} else {
@ -72,7 +73,7 @@ func TestBucket(t *testing.T) {
tomb.Go(func() error {
wg.Add(1)
defer wg.Done()
if err := testOneBucket(t, fname, tomb); err != nil {
if err := testOneBucket(t, hub, fname, tomb); err != nil {
t.Fatalf("Test '%s' failed : %s", fname, err)
}
return nil
@ -94,7 +95,7 @@ func watchTomb(tomb *tomb.Tomb) {
}
}
func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
func testOneBucket(t *testing.T, hub *cwhub.Hub, dir string, tomb *tomb.Tomb) error {
var (
holders []BucketFactory
@ -131,7 +132,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
}
cscfg := &csconfig.CrowdsecServiceCfg{}
holders, response, err := LoadBuckets(cscfg, "tests", files, tomb, buckets, false)
holders, response, err := LoadBuckets(cscfg, hub, files, tomb, buckets, false)
if err != nil {
t.Fatalf("failed loading bucket : %s", err)
}

9
pkg/leakybucket/manager_load.go
View file

@ -179,17 +179,12 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
return nil
}
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, hub *cwhub.Hub, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
var (
ret = []BucketFactory{}
response chan types.Event
)
hub, err := cwhub.GetHub()
if err != nil {
return nil, nil, err
}
response = make(chan types.Event, 1)
for _, f := range files {
log.Debugf("Loading '%s'", f)
@ -217,7 +212,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []str
log.Tracef("End of yaml file")
break
}
bucketFactory.DataDir = dataDir
bucketFactory.DataDir = hub.GetDataDir()
//check empty
if bucketFactory.Name == "" {
log.Errorf("Won't load nameless bucket")

36
pkg/setup/install.go
View file

@ -62,14 +62,19 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
if len(install.Collections) > 0 {
for _, collection := range setupItem.Install.Collections {
item := hub.GetItem(cwhub.COLLECTIONS, collection)
if item == nil {
return fmt.Errorf("collection %s not found", collection)
}
if dryRun {
fmt.Println("dry-run: would install collection", collection)
continue
}
if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", collection, err)
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", item.Name, err)
}
}
}
@ -82,8 +87,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
continue
}
if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", parser, err)
item := hub.GetItem(cwhub.PARSERS, parser)
if item == nil {
return fmt.Errorf("parser %s not found", parser)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", item.Name, err)
}
}
}
@ -96,8 +106,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
continue
}
if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", scenario, err)
item := hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
return fmt.Errorf("scenario %s not found", scenario)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", item.Name, err)
}
}
}
@ -110,8 +125,13 @@ func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
continue
}
if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
item := hub.GetItem(cwhub.POSTOVERFLOWS, postoverflow)
if item == nil {
return fmt.Errorf("postoverflow %s not found", postoverflow)
}
if err := item.Install(forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", item.Name, err)
}
}
}

25
test/bats/07_setup.bats
View file

@ -519,6 +519,11 @@ update-notifier-motd.timer enabled enabled
rune -0 cscli collections list -o json
rune -0 jq -r '.collections[].name' <(output)
refute_line "crowdsecurity/apache2"
# same with dependencies
rune -0 cscli collections remove --all
rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/linux"]}}]}'
assert_output 'dry-run: would install collection crowdsecurity/linux'
}
@test "cscli setup install-hub (dry run: install multiple collections)" {
@ -538,15 +543,17 @@ update-notifier-motd.timer enabled enabled
}
@test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" {
rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo","johndoe/bar"],"parsers":["crowdsecurity/fooparser","johndoe/barparser"],"scenarios":["crowdsecurity/fooscenario","johndoe/barscenario"],"postoverflows":["crowdsecurity/foopo","johndoe/barpo"]}}]}'
assert_line 'dry-run: would install collection crowdsecurity/foo'
assert_line 'dry-run: would install collection johndoe/bar'
assert_line 'dry-run: would install parser crowdsecurity/fooparser'
assert_line 'dry-run: would install parser johndoe/barparser'
assert_line 'dry-run: would install scenario crowdsecurity/fooscenario'
assert_line 'dry-run: would install scenario johndoe/barscenario'
assert_line 'dry-run: would install postoverflow crowdsecurity/foopo'
assert_line 'dry-run: would install postoverflow johndoe/barpo'
rune -0 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/aws-console","crowdsecurity/caddy"],"parsers":["crowdsecurity/asterisk-logs"],"scenarios":["crowdsecurity/smb-fs"],"postoverflows":["crowdsecurity/cdn-whitelist","crowdsecurity/rdns"]}}]}'
assert_line 'dry-run: would install collection crowdsecurity/aws-console'
assert_line 'dry-run: would install collection crowdsecurity/caddy'
assert_line 'dry-run: would install parser crowdsecurity/asterisk-logs'
assert_line 'dry-run: would install scenario crowdsecurity/smb-fs'
assert_line 'dry-run: would install postoverflow crowdsecurity/cdn-whitelist'
assert_line 'dry-run: would install postoverflow crowdsecurity/rdns'
rune -1 cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo"]}}]}'
assert_stderr --partial 'collection crowdsecurity/foo not found'
}
@test "cscli setup datasources" {

4
test/bats/20_hub.bats
View file

@ -66,8 +66,8 @@ teardown() {
new_hub=$(jq <"$HUB_DIR/.index.json" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")')
echo "$new_hub" >"$HUB_DIR/.index.json"
rune -0 cscli hub list --error
assert_stderr --partial "Referred parsers crowdsecurity/smb-logs in collection crowdsecurity/smb doesn't exist."
assert_stderr --partial "Referred scenarios crowdsecurity/mysql-bf in collection crowdsecurity/mysql doesn't exist."
assert_stderr --partial "can't find crowdsecurity/smb-logs in parsers, required by crowdsecurity/smb"
assert_stderr --partial "can't find crowdsecurity/mysql-bf in scenarios, required by crowdsecurity/mysql"
}
@test "cscli hub update" {

4
test/bats/20_hub_collections.bats
View file

@ -185,7 +185,7 @@ teardown() {
rune -0 cscli collections inspect crowdsecurity/sshd -o json
rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output)
# XXX: .installed is missing -- not false
assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null]'
assert_json '["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false]'
# one item, raw
rune -0 cscli collections inspect crowdsecurity/sshd -o raw
@ -211,7 +211,7 @@ teardown() {
# multiple items, json
rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o json
rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output)
assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",null],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",null]]'
assert_json '[["collections","crowdsecurity/sshd","crowdsecurity","collections/crowdsecurity/sshd.yaml",false],["collections","crowdsecurity/smb","crowdsecurity","collections/crowdsecurity/smb.yaml",false]]'
# multiple items, raw
rune -0 cscli collections inspect crowdsecurity/sshd crowdsecurity/smb -o raw

2
test/bats/20_hub_items.bats
View file

@ -53,7 +53,7 @@ teardown() {
rune -0 cscli collections inspect crowdsecurity/sshd -o json
# XXX: is this supposed to be tainted or up to date?
rune -0 jq -c '[.local_version,.up_to_date,.tainted]' <(output)
assert_json '["1.10",null,null]'
assert_json '["1.10",false,false]'
}
@test "hub index with invalid (non semver) version numbers" {

5
test/bats/20_hub_parsers.bats
View file

@ -187,8 +187,7 @@ teardown() {
# one item, json
rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o json
rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output)
# XXX: .installed is missing -- not false
assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null]'
assert_json '["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false]'
# one item, raw
rune -0 cscli parsers inspect crowdsecurity/sshd-logs -o raw
@ -215,7 +214,7 @@ teardown() {
# multiple items, json
rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o json
rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output)
assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",null],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",null]]'
assert_json '[["parsers","s01-parse","crowdsecurity/sshd-logs","crowdsecurity","parsers/s01-parse/crowdsecurity/sshd-logs.yaml",false],["parsers","s02-enrich","crowdsecurity/whitelists","crowdsecurity","parsers/s02-enrich/crowdsecurity/whitelists.yaml",false]]'
# multiple items, raw
rune -0 cscli parsers inspect crowdsecurity/sshd-logs crowdsecurity/whitelists -o raw

4
test/bats/20_hub_postoverflows.bats
View file

@ -189,7 +189,7 @@ teardown() {
rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
rune -0 jq -c '[.type, .stage, .name, .author, .path, .installed]' <(output)
# XXX: .installed is missing -- not false
assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null]'
assert_json '["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false]'
# one item, raw
rune -0 cscli postoverflows inspect crowdsecurity/rdns -o raw
@ -216,7 +216,7 @@ teardown() {
# multiple items, json
rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o json
rune -0 jq -sc '[.[] | [.type, .stage, .name, .author, .path, .installed]]' <(output)
assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",null],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",null]]'
assert_json '[["postoverflows","s00-enrich","crowdsecurity/rdns","crowdsecurity","postoverflows/s00-enrich/crowdsecurity/rdns.yaml",false],["postoverflows","s01-whitelist","crowdsecurity/cdn-whitelist","crowdsecurity","postoverflows/s01-whitelist/crowdsecurity/cdn-whitelist.yaml",false]]'
# multiple items, raw
rune -0 cscli postoverflows inspect crowdsecurity/rdns crowdsecurity/cdn-whitelist -o raw

5
test/bats/20_hub_scenarios.bats
View file

@ -187,8 +187,7 @@ teardown() {
# one item, json
rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json
rune -0 jq -c '[.type, .name, .author, .path, .installed]' <(output)
# XXX: .installed is missing -- not false
assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null]'
assert_json '["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false]'
# one item, raw
rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o raw
@ -214,7 +213,7 @@ teardown() {
# multiple items, json
rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o json
rune -0 jq -sc '[.[] | [.type, .name, .author, .path, .installed]]' <(output)
assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",null],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",null]]'
assert_json '[["scenarios","crowdsecurity/ssh-bf","crowdsecurity","scenarios/crowdsecurity/ssh-bf.yaml",false],["scenarios","crowdsecurity/telnet-bf","crowdsecurity","scenarios/crowdsecurity/telnet-bf.yaml",false]]'
# multiple items, raw
rune -0 cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/telnet-bf -o raw