beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor configuration management (#698)

Browse source
This commit is contained in:
AlteredCoder 2021-03-24 18:16:17 +01:00 committed by GitHub
parent bf192593b6
commit 1e899c2211
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
63 changed files with 2014 additions and 721 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/crowdsec-cli/alerts.go
View file

@ -199,6 +199,9 @@ func NewAlertsCmd() *cobra.Command {
Args: cobra.MinimumNArgs(1),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
var err error
if err := csConfig.LoadAPIClient(); err != nil {
log.Fatalf("loading api client: %s", err.Error())
}
if csConfig.API.Client == nil {
log.Fatalln("There is no configuration on 'api_client:'")
}

3
cmd/crowdsec-cli/bouncers.go
View file

@ -31,6 +31,9 @@ To list/add/delete bouncers
Args: cobra.MinimumNArgs(1),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
var err error
if err := csConfig.LoadDBConfig(); err != nil {
log.Fatalf(err.Error())
}
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
log.Fatalf("unable to create new database client: %s", err)

12
cmd/crowdsec-cli/capi.go
View file

@ -28,11 +28,14 @@ func NewCapiCmd() *cobra.Command {
Short: "Manage interaction with Central API (CAPI)",
Args: cobra.MinimumNArgs(1),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.API.Server == nil {
log.Fatalln("There is no API->server configuration")
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.Self)
log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.FilePath)
}
return nil
@ -124,7 +127,12 @@ func NewCapiCmd() *cobra.Command {
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}

32
cmd/crowdsec-cli/collections.go
View file

@ -18,13 +18,22 @@ func NewCollectionsCmd() *cobra.Command {
/*TBD fix help*/
Args: cobra.MinimumNArgs(1),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
@ -42,10 +51,6 @@ func NewCollectionsCmd() *cobra.Command {
Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
for _, name := range args {
InstallItem(name, cwhub.COLLECTIONS, forceAction)
}
@ -62,11 +67,6 @@ func NewCollectionsCmd() *cobra.Command {
Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
RemoveMany(cwhub.COLLECTIONS, "")
} else {
@ -95,10 +95,6 @@ func NewCollectionsCmd() *cobra.Command {
Long: `Fetch and upgrade given collection(s) from hub`,
Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
UpgradeConfig(cwhub.COLLECTIONS, "", forceAction)
} else {
@ -119,10 +115,6 @@ func NewCollectionsCmd() *cobra.Command {
Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
for _, name := range args {
InspectItem(name, cwhub.COLLECTIONS)
}
@ -138,10 +130,6 @@ func NewCollectionsCmd() *cobra.Command {
Example: `cscli collections list`,
Args: cobra.ExactArgs(0),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
ListItem(cwhub.COLLECTIONS, args)
},
}

13
cmd/crowdsec-cli/config.go
View file

@ -44,7 +44,7 @@ func backupConfigToDirectory(dirPath string) error {
return errors.Wrapf(err, "while checking parent directory %s existence", parentDir)
}
if err = os.Mkdir(dirPath, 0600); err != nil {
if err = os.Mkdir(dirPath, 0700); err != nil {
return fmt.Errorf("error while creating %s : %s", dirPath, err)
}
@ -68,7 +68,7 @@ func backupConfigToDirectory(dirPath string) error {
}
acquisBackupDir := dirPath + "/acquis/"
if err = os.Mkdir(acquisBackupDir, 0600); err != nil {
if err = os.Mkdir(acquisBackupDir, 0700); err != nil {
return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err)
}
@ -215,7 +215,7 @@ func restoreConfigFromDirectory(dirPath string) error {
/*if there is a acquisition dir, restore its content*/
if csConfig.Crowdsec.AcquisitionDirPath != "" {
if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0600); err != nil {
if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0700); err != nil {
return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
}
@ -389,7 +389,10 @@ func NewConfigCmd() *cobra.Command {
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var err error
if err = cwhub.GetHubIdx(csConfig.Cscli); err != nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if err = cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
@ -414,7 +417,7 @@ func NewConfigCmd() *cobra.Command {
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
var err error
if err = cwhub.GetHubIdx(csConfig.Cscli); err != nil {
if err = cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}

3
cmd/crowdsec-cli/decisions.go
View file

@ -105,6 +105,9 @@ func NewDecisionsCmd() *cobra.Command {
/*TBD example*/
Args: cobra.MinimumNArgs(1),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadAPIClient(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.API.Client == nil {
log.Fatalln("There is no configuration on 'api_client:'")
}

15
cmd/crowdsec-cli/hub.go
View file

@ -40,7 +40,10 @@ cscli hub update # Download list of available configurations from the hub
Short: "List installed configs",
Args: cobra.ExactArgs(0),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
@ -77,7 +80,10 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
return nil
},
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.UpdateHubIdx(csConfig.Cscli); err != nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
},
@ -102,7 +108,10 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
return nil
},
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}

11
cmd/crowdsec-cli/lapi.go
View file

@ -28,11 +28,14 @@ func NewLapiCmd() *cobra.Command {
Short: "Manage interaction with Local API (LAPI)",
Args: cobra.MinimumNArgs(1),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %s", err.Error())
}
if csConfig.API.Client == nil {
log.Fatalln("There is no API->client configuration")
}
if csConfig.API.Client.Credentials == nil {
log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.Self)
log.Fatalf("no configuration for crowdsec API in '%s'", *csConfig.FilePath)
}
return nil
},
@ -133,7 +136,11 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
}
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}

9
cmd/crowdsec-cli/machines.go
View file

@ -84,9 +84,15 @@ func NewMachinesCmd() *cobra.Command {
Long: `
Machines Management.
To list/add/delete/register/validate machines
To list/add/delete/validate machines
`,
Example: `cscli machines [action]`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadDBConfig(); err != nil {
log.Fatalf(err.Error())
}
return nil
},
}
var cmdMachinesList = &cobra.Command{
@ -97,6 +103,7 @@ To list/add/delete/register/validate machines
Args: cobra.MaximumNArgs(1),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
log.Fatalf("unable to create new database client: %s", err)

15
cmd/crowdsec-cli/main.go
View file

@ -14,7 +14,7 @@ import (
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
var ConfigFilePath string
var csConfig *csconfig.GlobalConfig
var csConfig *csconfig.Config
var dbClient *database.Client
var OutputFormat string
@ -28,7 +28,7 @@ var restoreOldBackup bool
var prometheusURL string
func initConfig() {
var err error
if trace_lvl {
log.SetLevel(log.TraceLevel)
} else if dbg_lvl {
@ -42,12 +42,15 @@ func initConfig() {
}
logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 03:04:05 PM", FullTimestamp: true}
log.SetFormatter(logFormatter)
csConfig = csconfig.NewConfig()
log.Debugf("Using %s as configuration file", ConfigFilePath)
if err := csConfig.LoadConfigurationFile(ConfigFilePath, csConfig.DisableAPI, csConfig.DisableAgent); err != nil {
csConfig, err = csconfig.NewConfig(ConfigFilePath, false, false)
if err != nil {
log.Fatalf(err.Error())
}
log.Debugf("Using %s as configuration file", ConfigFilePath)
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.Cscli == nil {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
}

5
cmd/crowdsec-cli/metrics.go
View file

@ -377,6 +377,9 @@ func NewMetricsCmd() *cobra.Command {
Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`,
Args: cobra.ExactArgs(0),
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadPrometheus(); err != nil {
log.Fatalf(err.Error())
}
if !csConfig.Prometheus.Enabled {
log.Warningf("Prometheus is not enabled, can't show metrics")
os.Exit(1)
@ -387,7 +390,7 @@ func NewMetricsCmd() *cobra.Command {
}
if prometheusURL == "" {
log.Errorf("No prometheus url, please specify in %s or via -u", *csConfig.Self)
log.Errorf("No prometheus url, please specify in %s or via -u", *csConfig.FilePath)
os.Exit(1)
}

32
cmd/crowdsec-cli/parsers.go
View file

@ -22,13 +22,21 @@ cscli parsers remove crowdsecurity/sshd-logs
`,
Args: cobra.MinimumNArgs(1),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
@ -46,10 +54,6 @@ cscli parsers remove crowdsecurity/sshd-logs
Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
for _, name := range args {
InstallItem(name, cwhub.PARSERS, forceAction)
}
@ -66,11 +70,6 @@ cscli parsers remove crowdsecurity/sshd-logs
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
RemoveMany(cwhub.PARSERS, "")
} else {
@ -91,10 +90,6 @@ cscli parsers remove crowdsecurity/sshd-logs
Long: `Fetch and upgrade given parser(s) from hub`,
Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
UpgradeConfig(cwhub.PARSERS, "", forceAction)
} else {
@ -115,11 +110,6 @@ cscli parsers remove crowdsecurity/sshd-logs
Example: `cscli parsers inspect crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
InspectItem(args[0], cwhub.PARSERS)
},
}
@ -133,10 +123,6 @@ cscli parsers remove crowdsecurity/sshd-logs
Example: `cscli parsers list
cscli parser list crowdsecurity/xxx`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
ListItem(cwhub.PARSERS, args)
},
}

31
cmd/crowdsec-cli/postoverflows.go
View file

@ -21,13 +21,21 @@ func NewPostOverflowsCmd() *cobra.Command {
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
@ -45,10 +53,6 @@ func NewPostOverflowsCmd() *cobra.Command {
Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
for _, name := range args {
InstallItem(name, cwhub.PARSERS_OVFLW, forceAction)
}
@ -65,11 +69,6 @@ func NewPostOverflowsCmd() *cobra.Command {
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
RemoveMany(cwhub.PARSERS_OVFLW, "")
} else {
@ -90,10 +89,6 @@ func NewPostOverflowsCmd() *cobra.Command {
Long: `Fetch and Upgrade given postoverflow(s) from hub`,
Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
UpgradeConfig(cwhub.PARSERS_OVFLW, "", forceAction)
} else {
@ -114,10 +109,6 @@ func NewPostOverflowsCmd() *cobra.Command {
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
InspectItem(args[0], cwhub.PARSERS_OVFLW)
},
}
@ -130,10 +121,6 @@ func NewPostOverflowsCmd() *cobra.Command {
Example: `cscli postoverflows list
cscli postoverflows list crowdsecurity/xxx`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
ListItem(cwhub.PARSERS_OVFLW, args)
},
}

31
cmd/crowdsec-cli/scenarios.go
View file

@ -22,13 +22,21 @@ cscli scenarios remove crowdsecurity/ssh-bf
`,
Args: cobra.MinimumNArgs(1),
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := setHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
@ -46,10 +54,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
for _, name := range args {
InstallItem(name, cwhub.SCENARIOS, forceAction)
}
@ -66,11 +70,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
RemoveMany(cwhub.SCENARIOS, "")
} else {
@ -91,10 +90,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
Long: `Fetch and Upgrade given scenario(s) from hub`,
Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
if all {
UpgradeConfig(cwhub.SCENARIOS, "", forceAction)
} else {
@ -115,10 +110,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
Example: `cscli scenarios inspect crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
InspectItem(args[0], cwhub.SCENARIOS)
},
}
@ -132,10 +123,6 @@ cscli scenarios remove crowdsecurity/ssh-bf
Example: `cscli scenarios list
cscli scenarios list crowdsecurity/xxx`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
ListItem(cwhub.SCENARIOS, args)
},
}

63
cmd/crowdsec-cli/simulation.go
View file

@ -11,25 +11,25 @@ import (
)
func addToExclusion(name string) error {
csConfig.Crowdsec.SimulationConfig.Exclusions = append(csConfig.Crowdsec.SimulationConfig.Exclusions, name)
csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
return nil
}
func removeFromExclusion(name string) error {
index := indexOf(name, csConfig.Crowdsec.SimulationConfig.Exclusions)
index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
// Remove element from the slice
csConfig.Crowdsec.SimulationConfig.Exclusions[index] = csConfig.Crowdsec.SimulationConfig.Exclusions[len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1]
csConfig.Crowdsec.SimulationConfig.Exclusions[len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1] = ""
csConfig.Crowdsec.SimulationConfig.Exclusions = csConfig.Crowdsec.SimulationConfig.Exclusions[:len(csConfig.Crowdsec.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
return nil
}
func enableGlobalSimulation() error {
csConfig.Crowdsec.SimulationConfig.Simulation = new(bool)
*csConfig.Crowdsec.SimulationConfig.Simulation = true
csConfig.Crowdsec.SimulationConfig.Exclusions = []string{}
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = true
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("unable to dump simulation file: %s", err.Error())
@ -41,7 +41,7 @@ func enableGlobalSimulation() error {
}
func dumpSimulationFile() error {
newConfigSim, err := yaml.Marshal(csConfig.Crowdsec.SimulationConfig)
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
@ -55,11 +55,11 @@ func dumpSimulationFile() error {
}
func disableGlobalSimulation() error {
csConfig.Crowdsec.SimulationConfig.Simulation = new(bool)
*csConfig.Crowdsec.SimulationConfig.Simulation = false
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = false
csConfig.Crowdsec.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(csConfig.Crowdsec.SimulationConfig)
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
@ -73,23 +73,23 @@ func disableGlobalSimulation() error {
}
func simulationStatus() error {
if csConfig.Crowdsec.SimulationConfig == nil {
if csConfig.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return nil
}
if *csConfig.Crowdsec.SimulationConfig.Simulation {
if *csConfig.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(csConfig.Crowdsec.SimulationConfig.Exclusions) > 0 {
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range csConfig.Crowdsec.SimulationConfig.Exclusions {
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(csConfig.Crowdsec.SimulationConfig.Exclusions) > 0 {
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range csConfig.Crowdsec.SimulationConfig.Exclusions {
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
@ -105,9 +105,15 @@ func NewSimulationCmds() *cobra.Command {
cscli simulation enable crowdsecurity/ssh-bf
cscli simulation disable crowdsecurity/ssh-bf`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadSimulation(); err != nil {
log.Fatalf(err.Error())
}
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before using simulation")
}
if csConfig.Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
@ -125,7 +131,10 @@ cscli simulation disable crowdsecurity/ssh-bf`,
Short: "Enable the simulation, globally or on specified scenarios",
Example: `cscli simulation enable`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(csConfig.Cscli); err != nil {
if err := csConfig.LoadHub(); err != nil {
log.Fatalf(err.Error())
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
log.Infoln("Run 'sudo cscli hub update' to get the hub index")
}
@ -143,16 +152,16 @@ cscli simulation disable crowdsecurity/ssh-bf`,
if !item.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
isExcluded := inSlice(scenario, csConfig.Crowdsec.SimulationConfig.Exclusions)
if *csConfig.Crowdsec.SimulationConfig.Simulation && !isExcluded {
isExcluded := inSlice(scenario, csConfig.Cscli.SimulationConfig.Exclusions)
if *csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warningf("global simulation is already enabled")
continue
}
if !*csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
log.Warningf("simulation for '%s' already enabled", scenario)
continue
}
if *csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
if *csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatalf(err.Error())
}
@ -186,12 +195,12 @@ cscli simulation disable crowdsecurity/ssh-bf`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
for _, scenario := range args {
isExcluded := inSlice(scenario, csConfig.Crowdsec.SimulationConfig.Exclusions)
if !*csConfig.Crowdsec.SimulationConfig.Simulation && !isExcluded {
isExcluded := inSlice(scenario, csConfig.Cscli.SimulationConfig.Exclusions)
if !*csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warningf("%s isn't in simulation mode", scenario)
continue
}
if !*csConfig.Crowdsec.SimulationConfig.Simulation && isExcluded {
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatalf(err.Error())
}

18
cmd/crowdsec-cli/utils.go
View file

@ -141,16 +141,16 @@ func InstallItem(name string, obtype string, force bool) {
return
}
}
item, err := cwhub.DownloadLatest(csConfig.Cscli, item, force)
item, err := cwhub.DownloadLatest(csConfig.Hub, item, force)
if err != nil {
log.Fatalf("error while downloading %s : %v", item.Name, err)
}
cwhub.AddItem(obtype, item)
if downloadOnly {
log.Infof("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath)
log.Infof("Downloaded %s to %s", item.Name, csConfig.Hub.HubDir+"/"+item.RemotePath)
return
}
item, err = cwhub.EnableItem(csConfig.Cscli, item)
item, err = cwhub.EnableItem(csConfig.Hub, item)
if err != nil {
log.Fatalf("error while enabled %s : %v.", item.Name, err)
}
@ -168,7 +168,7 @@ func RemoveMany(itemType string, name string) {
log.Fatalf("unable to retrieve: %s", name)
}
item := *it
item, err = cwhub.DisableItem(csConfig.Cscli, item, purge, forceAction)
item, err = cwhub.DisableItem(csConfig.Hub, item, purge, forceAction)
if err != nil {
log.Fatalf("unable to disable %s : %v", item.Name, err)
}
@ -176,7 +176,7 @@ func RemoveMany(itemType string, name string) {
return
} else if name == "" && all {
for _, v := range cwhub.GetItemMap(itemType) {
v, err = cwhub.DisableItem(csConfig.Cscli, v, purge, forceAction)
v, err = cwhub.DisableItem(csConfig.Hub, v, purge, forceAction)
if err != nil {
log.Fatalf("unable to disable %s : %v", v.Name, err)
}
@ -219,7 +219,7 @@ func UpgradeConfig(itemType string, name string, force bool) {
continue
}
}
v, err = cwhub.DownloadLatest(csConfig.Cscli, v, force)
v, err = cwhub.DownloadLatest(csConfig.Hub, v, force)
if err != nil {
log.Fatalf("%s : download failed : %v", v.Name, err)
}
@ -264,7 +264,7 @@ func InspectItem(name string, objecitemType string) {
fmt.Printf("%s", string(buff))
if csConfig.Prometheus.Enabled {
if csConfig.Prometheus.ListenAddr == "" || csConfig.Prometheus.ListenPort == 0 {
log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.Self)
log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.FilePath)
return
}
if prometheusURL == "" {
@ -500,7 +500,7 @@ func silenceInstallItem(name string, obtype string) (string, error) {
if downloadOnly && it.Downloaded && it.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
}
it, err := cwhub.DownloadLatest(csConfig.Cscli, it, forceAction)
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
}
@ -511,7 +511,7 @@ func silenceInstallItem(name string, obtype string) (string, error) {
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
}
it, err = cwhub.EnableItem(csConfig.Cscli, it)
it, err = cwhub.EnableItem(csConfig.Hub, it)
if err != nil {
return "", fmt.Errorf("error while enabled %s : %v", it.Name, err)
}

2
cmd/crowdsec/api.go
View file

@ -9,7 +9,7 @@ import (
log "github.com/sirupsen/logrus"
)
func initAPIServer(cConfig *csconfig.GlobalConfig) (*apiserver.APIServer, error) {
func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
apiServer, err := apiserver.NewServer(cConfig.API.Server)
if err != nil {
return nil, fmt.Errorf("unable to run local API: %s", err)

8
cmd/crowdsec/crowdsec.go
View file

@ -14,14 +14,14 @@ import (
log "github.com/sirupsen/logrus"
)
func initCrowdsec(cConfig *csconfig.GlobalConfig) (*parser.Parsers, error) {
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
err := exprhelpers.Init()
if err != nil {
return &parser.Parsers{}, fmt.Errorf("Failed to init expr helpers : %s", err)
}
// Populate cwhub package tools
if err := cwhub.GetHubIdx(cConfig.Cscli); err != nil {
if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
}
@ -41,7 +41,7 @@ func initCrowdsec(cConfig *csconfig.GlobalConfig) (*parser.Parsers, error) {
return csParsers, nil
}
func runCrowdsec(cConfig *csconfig.GlobalConfig, parsers *parser.Parsers) error {
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
inputLineChan := make(chan types.Event)
inputEventChan := make(chan types.Event)
@ -117,7 +117,7 @@ func runCrowdsec(cConfig *csconfig.GlobalConfig, parsers *parser.Parsers) error
return nil
}
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.GlobalConfig) {
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config) {
crowdsecTomb.Go(func() error {
defer types.CatchPanic("crowdsec/serveCrowdsec")
go func() {

59
cmd/crowdsec/main.go
View file

@ -33,9 +33,6 @@ var (
apiTomb tomb.Tomb
crowdsecTomb tomb.Tomb
disableAPI bool
disableAgent bool
flags *Flags
/*the state of acquisition*/
@ -112,7 +109,7 @@ func newParsers() *parser.Parsers {
return parsers
}
func LoadBuckets(cConfig *csconfig.GlobalConfig) error {
func LoadBuckets(cConfig *csconfig.Config) error {
var (
err error
@ -140,7 +137,7 @@ func LoadBuckets(cConfig *csconfig.GlobalConfig) error {
return nil
}
func LoadAcquisition(cConfig *csconfig.GlobalConfig) error {
func LoadAcquisition(cConfig *csconfig.Config) error {
var err error
if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
@ -191,31 +188,25 @@ func (f *Flags) Parse() {
}
// LoadConfig return configuration parsed from configuration file
func LoadConfig(cConfig *csconfig.GlobalConfig) error {
disableAPI = flags.DisableAPI
disableAgent = flags.DisableAgent
if flags.ConfigFile != "" {
if err := cConfig.LoadConfigurationFile(flags.ConfigFile, disableAPI, disableAgent); err != nil {
return fmt.Errorf("while loading configuration : %s", err)
func LoadConfig(cConfig *csconfig.Config) error {
if !flags.DisableAgent {
if err := cConfig.LoadCrowdsec(); err != nil {
return err
}
} else {
log.Warningf("no configuration file provided")
}
if !disableAPI && (cConfig.API == nil || cConfig.API.Server == nil) {
log.Errorf("no API server configuration found, will not start the local API")
disableAPI = true
}
if !disableAgent && cConfig.Crowdsec == nil {
log.Errorf("no configuration found crowdsec agent, will not start the agent")
disableAgent = true
if !flags.DisableAPI {
if err := cConfig.LoadAPIServer(); err != nil {
return err
}
}
if !disableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
if !cConfig.DisableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
log.Fatalf("missing local API credentials for crowdsec agent, abort")
}
if disableAPI && disableAgent {
if cConfig.DisableAPI && cConfig.DisableAgent {
log.Fatalf("You must run at least the API Server or crowdsec")
}
@ -244,14 +235,14 @@ func LoadConfig(cConfig *csconfig.GlobalConfig) error {
cConfig.Common.LogLevel = &logLevel
}
if flags.TestMode && !disableAgent {
if flags.TestMode && !cConfig.DisableAgent {
cConfig.Crowdsec.LintOnly = true
}
if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
cConfig.API.Server.OnlineClient = nil
/*if the api is disabled as well, just read file and exit, don't daemonize*/
if disableAPI {
if flags.DisableAPI {
cConfig.Common.Daemonize = false
}
cConfig.Common.LogMedia = "stdout"
@ -263,13 +254,12 @@ func LoadConfig(cConfig *csconfig.GlobalConfig) error {
func main() {
var (
cConfig *csconfig.GlobalConfig
cConfig *csconfig.Config
err error
)
defer types.CatchPanic("crowdsec/main")
cConfig = csconfig.NewConfig()
// Handle command line arguments
flags = &Flags{}
flags.Parse()
@ -278,6 +268,10 @@ func main() {
os.Exit(0)
}
cConfig, err = csconfig.NewConfig(flags.ConfigFile, flags.DisableAgent, flags.DisableAPI)
if err != nil {
log.Fatalf(err.Error())
}
if err := LoadConfig(cConfig); err != nil {
log.Fatalf(err.Error())
}
@ -288,19 +282,6 @@ func main() {
log.Infof("Crowdsec %s", cwversion.VersionStr())
if !flags.DisableAPI && (cConfig.API == nil || cConfig.API.Server == nil) {
log.Errorf("no API server configuration found, will not start the local API")
flags.DisableAPI = true
}
if !flags.DisableAgent && cConfig.Crowdsec == nil {
log.Errorf("no configuration found crowdsec agent, will not start the agent")
flags.DisableAgent = true
}
if !flags.DisableAgent && (cConfig.API == nil || cConfig.API.Client == nil || cConfig.API.Client.Credentials == nil) {
log.Fatalf("missing local API credentials for crowdsec agent, abort")
}
// Enable profiling early
if cConfig.Prometheus != nil {
go registerPrometheus(cConfig.Prometheus)

2
cmd/crowdsec/pour.go
View file

@ -10,7 +10,7 @@ import (
log "github.com/sirupsen/logrus"
)
func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.GlobalConfig) error {
func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error {
var (
count int
)

20
cmd/crowdsec/serve.go
View file

@ -19,7 +19,7 @@ import (
)
//debugHandler is kept as a dev convenience : it shuts down and serialize internal state
func debugHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
func debugHandler(sig os.Signal, cConfig *csconfig.Config) error {
var tmpFile string
var err error
//stop go routines
@ -37,12 +37,12 @@ func debugHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
return nil
}
func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
func reloadHandler(sig os.Signal, cConfig *csconfig.Config) error {
var tmpFile string
var err error
//stop go routines
if !disableAgent {
if !cConfig.DisableAgent {
if err := shutdownCrowdsec(); err != nil {
log.Fatalf("Failed to shut down crowdsec routines: %s", err)
}
@ -57,7 +57,7 @@ func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
}
}
if !disableAPI {
if !cConfig.DisableAPI {
if err := shutdownAPI(); err != nil {
log.Fatalf("Failed to shut down api routines: %s", err)
}
@ -81,7 +81,7 @@ func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
log.Fatal(err.Error())
}
if !disableAPI {
if !cConfig.DisableAPI {
apiServer, err := initAPIServer(cConfig)
if err != nil {
return fmt.Errorf("unable to init api server: %s", err)
@ -90,7 +90,7 @@ func reloadHandler(sig os.Signal, cConfig *csconfig.GlobalConfig) error {
serveAPIServer(apiServer)
}
if !disableAgent {
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
if err != nil {
return fmt.Errorf("unable to init crowdsec: %s", err)
@ -186,7 +186,7 @@ func termHandler(sig os.Signal) error {
return nil
}
func HandleSignals(cConfig *csconfig.GlobalConfig) {
func HandleSignals(cConfig *csconfig.Config) {
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan,
syscall.SIGHUP,
@ -220,7 +220,7 @@ func HandleSignals(cConfig *csconfig.GlobalConfig) {
os.Exit(code)
}
func Serve(cConfig *csconfig.GlobalConfig) error {
func Serve(cConfig *csconfig.Config) error {
acquisTomb = tomb.Tomb{}
parsersTomb = tomb.Tomb{}
bucketsTomb = tomb.Tomb{}
@ -228,7 +228,7 @@ func Serve(cConfig *csconfig.GlobalConfig) error {
apiTomb = tomb.Tomb{}
crowdsecTomb = tomb.Tomb{}
if !disableAPI {
if !cConfig.DisableAPI {
apiServer, err := initAPIServer(cConfig)
if err != nil {
return errors.Wrap(err, "api server init")
@ -238,7 +238,7 @@ func Serve(cConfig *csconfig.GlobalConfig) error {
}
}
if !disableAgent {
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
if err != nil {
return errors.Wrap(err, "crowdsec init")

8
pkg/apiserver/apiserver_test.go
View file

@ -31,8 +31,8 @@ var MachineTest = models.WatcherAuthRequest{
var UserAgent = fmt.Sprintf("crowdsec-test/%s", cwversion.Version)
func LoadTestConfig() csconfig.GlobalConfig {
config := csconfig.GlobalConfig{}
func LoadTestConfig() csconfig.Config {
config := csconfig.Config{}
maxAge := "1h"
flushConfig := csconfig.FlushDBCfg{
MaxAge: &maxAge,
@ -57,8 +57,8 @@ func LoadTestConfig() csconfig.GlobalConfig {
return config
}
func LoadTestConfigForwardedFor() csconfig.GlobalConfig {
config := csconfig.GlobalConfig{}
func LoadTestConfigForwardedFor() csconfig.Config {
config := csconfig.Config{}
maxAge := "1h"
flushConfig := csconfig.FlushDBCfg{
MaxAge: &maxAge,

93
pkg/csconfig/api.go
View file

@ -1,6 +1,15 @@
package csconfig
import log "github.com/sirupsen/logrus"
import (
"fmt"
"io/ioutil"
"strings"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
type APICfg struct {
Client *LocalApiClientCfg `yaml:"client"`
@ -26,6 +35,47 @@ type LocalApiClientCfg struct {
InsecureSkipVerify *bool `yaml:"insecure_skip_verify"` // check if api certificate is bad or not
}
func (o *OnlineApiClientCfg) Load() error {
o.Credentials = new(ApiCredentialsCfg)
fcontent, err := ioutil.ReadFile(o.CredentialsFilePath)
if err != nil {
return errors.Wrapf(err, "failed to read api server credentials configuration file '%s'", o.CredentialsFilePath)
}
err = yaml.UnmarshalStrict(fcontent, o.Credentials)
if err != nil {
return errors.Wrapf(err, "failed unmarshaling api server credentials configuration file '%s'", o.CredentialsFilePath)
}
if o.Credentials.Login == "" || o.Credentials.Password == "" || o.Credentials.URL == "" {
log.Warningf("can't load CAPI credentials from '%s' (missing field)", o.CredentialsFilePath)
o.Credentials = nil
}
return nil
}
func (l *LocalApiClientCfg) Load() error {
fcontent, err := ioutil.ReadFile(l.CredentialsFilePath)
if err != nil {
return errors.Wrapf(err, "failed to read api client credential configuration file '%s'", l.CredentialsFilePath)
}
err = yaml.UnmarshalStrict(fcontent, &l.Credentials)
if err != nil {
return errors.Wrapf(err, "failed unmarshaling api client credential configuration file '%s'", l.CredentialsFilePath)
}
if l.Credentials != nil && l.Credentials.URL != "" {
if !strings.HasSuffix(l.Credentials.URL, "/") {
l.Credentials.URL = l.Credentials.URL + "/"
}
} else {
log.Warningf("no credentials or URL found in api client configuration '%s'", l.CredentialsFilePath)
}
if l.InsecureSkipVerify == nil {
apiclient.InsecureSkipVerify = false
} else {
apiclient.InsecureSkipVerify = *l.InsecureSkipVerify
}
return nil
}
/*local api service configuration*/
type LocalApiServerCfg struct {
ListenURI string `yaml:"listen_uri,omitempty"` //127.0.0.1:8080
@ -44,3 +94,44 @@ type TLSCfg struct {
CertFilePath string `yaml:"cert_file"`
KeyFilePath string `yaml:"key_file"`
}
func (c *Config) LoadAPIServer() error {
if c.API.Server != nil && !c.DisableAPI {
if err := c.LoadCommon(); err != nil {
return fmt.Errorf("loading common configuration: %s", err.Error())
}
c.API.Server.LogDir = c.Common.LogDir
c.API.Server.LogMedia = c.Common.LogMedia
if err := c.API.Server.LoadProfiles(); err != nil {
return errors.Wrap(err, "while loading profiles for LAPI")
}
if c.API.Server.OnlineClient != nil && c.API.Server.OnlineClient.CredentialsFilePath != "" {
if err := c.API.Server.OnlineClient.Load(); err != nil {
return errors.Wrap(err, "loading online client credentials")
}
}
if c.API.Server.OnlineClient == nil || c.API.Server.OnlineClient.Credentials == nil {
log.Printf("push and pull to crowdsec API disabled")
}
if err := c.LoadDBConfig(); err != nil {
return err
}
} else {
log.Warningf("crowdsec local API is disabled")
c.DisableAPI = true
}
return nil
}
func (c *Config) LoadAPIClient() error {
if c.API != nil && c.API.Client != nil && c.API.Client.CredentialsFilePath != "" && !c.DisableAgent {
if err := c.API.Client.Load(); err != nil {
return err
}
} else {
return fmt.Errorf("no API client section in configuration")
}
return nil
}

268
pkg/csconfig/api_test.go Normal file
View file

@ -0,0 +1,268 @@
package csconfig
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
)
func TestLoadLocalApiClientCfg(t *testing.T) {
True := true
tests := []struct {
name string
Input *LocalApiClientCfg
expectedResult *ApiCredentialsCfg
err string
}{
{
name: "basic valid configuration",
Input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
},
expectedResult: &ApiCredentialsCfg{
URL: "http://localhost:8080/",
Login: "test",
Password: "testpassword",
},
},
{
name: "invalid configuration",
Input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
},
expectedResult: &ApiCredentialsCfg{},
},
{
name: "invalid configuration filepath",
Input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/nonexist_lapi-secrets.yaml",
},
expectedResult: nil,
},
{
name: "valid configuration with insecure skip verify",
Input: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
InsecureSkipVerify: &True,
},
expectedResult: &ApiCredentialsCfg{
URL: "http://localhost:8080/",
Login: "test",
Password: "testpassword",
},
},
}
for idx, test := range tests {
fmt.Printf("TEST '%s'\n", test.name)
err := test.Input.Load()
if err == nil && test.err != "" {
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Credentials)
if !isOk {
t.Fatalf("test '%s' failed", test.name)
}
}
}
func TestLoadOnlineApiClientCfg(t *testing.T) {
tests := []struct {
name string
Input *OnlineApiClientCfg
expectedResult *ApiCredentialsCfg
err string
}{
{
name: "basic valid configuration",
Input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/online-api-secrets.yaml",
},
expectedResult: &ApiCredentialsCfg{
URL: "http://crowdsec.api",
Login: "test",
Password: "testpassword",
},
},
{
name: "invalid configuration",
Input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/bad_lapi-secrets.yaml",
},
expectedResult: &ApiCredentialsCfg{},
err: "failed unmarshaling api server credentials",
},
{
name: "missing field configuration",
Input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/bad_online-api-secrets.yaml",
},
expectedResult: nil,
},
{
name: "invalid configuration filepath",
Input: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/nonexist_online-api-secrets.yaml",
},
expectedResult: &ApiCredentialsCfg{},
err: "failed to read api server credentials",
},
}
for idx, test := range tests {
err := test.Input.Load()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Credentials)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}
func TestLoadAPIServer(t *testing.T) {
tmpLAPI := &LocalApiServerCfg{
ProfilesPath: "./tests/profiles.yaml",
}
if err := tmpLAPI.LoadProfiles(); err != nil {
t.Fatalf("loading tmp profiles: %+v", err)
}
LogDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatalf(err.Error())
}
config := &Config{}
fcontent, err := ioutil.ReadFile("./tests/config.yaml")
if err != nil {
t.Fatalf(err.Error())
}
configData := os.ExpandEnv(string(fcontent))
err = yaml.UnmarshalStrict([]byte(configData), &config)
if err != nil {
t.Fatalf(err.Error())
}
tests := []struct {
name string
Input *Config
expectedResult *LocalApiServerCfg
err string
}{
{
name: "basic valid configuration",
Input: &Config{
Self: []byte(configData),
API: &APICfg{
Server: &LocalApiServerCfg{
ListenURI: "http://crowdsec.api",
OnlineClient: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/online-api-secrets.yaml",
},
ProfilesPath: "./tests/profiles.yaml",
},
},
DbConfig: &DatabaseCfg{
Type: "sqlite",
DbPath: "./tests/test.db",
},
Common: &CommonCfg{
LogDir: "./tests/",
LogMedia: "stdout",
},
DisableAPI: false,
},
expectedResult: &LocalApiServerCfg{
ListenURI: "http://crowdsec.api",
TLS: nil,
DbConfig: &DatabaseCfg{
DbPath: "./tests/test.db",
Type: "sqlite",
},
LogDir: LogDirFullPath,
LogMedia: "stdout",
OnlineClient: &OnlineApiClientCfg{
CredentialsFilePath: "./tests/online-api-secrets.yaml",
Credentials: &ApiCredentialsCfg{
URL: "http://crowdsec.api",
Login: "test",
Password: "testpassword",
},
},
Profiles: tmpLAPI.Profiles,
ProfilesPath: "./tests/profiles.yaml",
UseForwardedForHeaders: false,
},
err: "",
},
{
name: "basic valid configuration",
Input: &Config{
Self: []byte(configData),
API: &APICfg{
Server: &LocalApiServerCfg{},
},
Common: &CommonCfg{
LogDir: "./tests/",
LogMedia: "stdout",
},
DisableAPI: false,
},
expectedResult: &LocalApiServerCfg{
LogDir: LogDirFullPath,
LogMedia: "stdout",
},
err: "while loading profiles for LAPI",
},
}
for idx, test := range tests {
err := test.Input.LoadAPIServer()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.API.Server)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}

32
pkg/csconfig/common.go
View file

@ -1,6 +1,12 @@
package csconfig
import log "github.com/sirupsen/logrus"
import (
"fmt"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
/*daemonization/service related stuff*/
type CommonCfg struct {
@ -11,3 +17,27 @@ type CommonCfg struct {
LogLevel *log.Level `yaml:"log_level"`
WorkingDir string `yaml:"working_dir,omitempty"` ///var/run
}
func (c *Config) LoadCommon() error {
var err error
if c.Common == nil {
return fmt.Errorf("no common block provided in configuration file")
}
var CommonCleanup = []*string{
&c.Common.PidDir,
&c.Common.LogDir,
&c.Common.WorkingDir,
}
for _, k := range CommonCleanup {
if *k == "" {
continue
}
*k, err = filepath.Abs(*k)
if err != nil {
return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
}
}
return nil
}

98
pkg/csconfig/common_test.go Normal file
View file

@ -0,0 +1,98 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadCommon(t *testing.T) {
PidDirFullPath, err := filepath.Abs("./tests/")
if err != nil {
t.Fatalf(err.Error())
}
LogDirFullPath, err := filepath.Abs("./tests/log/")
if err != nil {
t.Fatalf(err.Error())
}
WorkingDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatalf(err.Error())
}
tests := []struct {
name string
Input *Config
expectedResult *CommonCfg
err string
}{
{
name: "basic valid configuration",
Input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./tests",
LogMedia: "file",
LogDir: "./tests/log/",
WorkingDir: "./tests/",
},
},
expectedResult: &CommonCfg{
Daemonize: true,
PidDir: PidDirFullPath,
LogMedia: "file",
LogDir: LogDirFullPath,
WorkingDir: WorkingDirFullPath,
},
},
{
name: "empty working dir",
Input: &Config{
Common: &CommonCfg{
Daemonize: true,
PidDir: "./tests",
LogMedia: "file",
LogDir: "./tests/log/",
},
},
expectedResult: &CommonCfg{
Daemonize: true,
PidDir: PidDirFullPath,
LogMedia: "file",
LogDir: LogDirFullPath,
},
},
{
name: "no common",
Input: &Config{},
expectedResult: nil,
},
}
for idx, test := range tests {
err := test.Input.LoadCommon()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Common)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}

262
pkg/csconfig/config.go
View file

@ -4,19 +4,17 @@ import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
/*top-level config : defaults,overriden by cfg file,overriden by cli*/
type GlobalConfig struct {
type Config struct {
//just a path to ourself :p
Self *string `yaml:"-"`
FilePath *string `yaml:"-"`
Self []byte `yaml:"-"`
Common *CommonCfg `yaml:"common,omitempty"`
Prometheus *PrometheusCfg `yaml:"prometheus,omitempty"`
Crowdsec *CrowdsecServiceCfg `yaml:"crowdsec_service,omitempty"`
@ -26,9 +24,10 @@ type GlobalConfig struct {
ConfigPaths *ConfigurationPaths `yaml:"config_paths,omitempty"`
DisableAPI bool `yaml:"-"`
DisableAgent bool `yaml:"-"`
Hub *Hub `yaml:"-"`
}
func (c *GlobalConfig) Dump() error {
func (c *Config) Dump() error {
out, err := yaml.Marshal(c)
if err != nil {
return errors.Wrap(err, "failed marshaling config")
@ -37,192 +36,26 @@ func (c *GlobalConfig) Dump() error {
return nil
}
func (c *GlobalConfig) LoadConfigurationFile(path string, disableAPI bool, disableAgent bool) error {
c.DisableAPI = disableAPI
c.DisableAgent = disableAgent
fcontent, err := ioutil.ReadFile(path)
func NewConfig(configFile string, disableAgent bool, disableAPI bool) (*Config, error) {
fcontent, err := ioutil.ReadFile(configFile)
if err != nil {
return errors.Wrap(err, "failed to read config file")
return nil, errors.Wrap(err, "failed to read config file")
}
configData := os.ExpandEnv(string(fcontent))
err = yaml.UnmarshalStrict([]byte(configData), c)
cfg := Config{
FilePath: &configFile,
DisableAgent: disableAgent,
DisableAPI: disableAPI,
}
err = yaml.UnmarshalStrict([]byte(configData), &cfg)
if err != nil {
return errors.Wrap(err, "failed unmarshaling config")
return nil, err
}
path, err = filepath.Abs(path)
if err != nil {
return errors.Wrap(err, "failed to load absolute path")
}
c.Self = &path
if err := c.LoadConfiguration(); err != nil {
return errors.Wrap(err, "failed to load sub configurations")
}
return nil
return &cfg, nil
}
func (c *GlobalConfig) LoadConfiguration() error {
if c.ConfigPaths.ConfigDir == "" {
return fmt.Errorf("please provide a configuration directory with the 'config_dir' directive in the 'config_paths' section")
}
if c.ConfigPaths.DataDir == "" {
return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
}
if c.ConfigPaths.HubDir == "" {
c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
}
if c.ConfigPaths.HubIndexFile == "" {
c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
}
if err := c.LoadSimulation(); err != nil {
return err
}
if c.Crowdsec != nil {
if c.Crowdsec.AcquisitionFilePath != "" {
log.Debugf("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath)
if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil {
return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath)
}
c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, c.Crowdsec.AcquisitionFilePath)
}
if c.Crowdsec.AcquisitionDirPath != "" {
files, err := filepath.Glob(c.Crowdsec.AcquisitionDirPath + "/*.yaml")
c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, files...)
if err != nil {
return errors.Wrap(err, "while globing acquis_dir")
}
}
if c.Crowdsec.AcquisitionDirPath == "" && c.Crowdsec.AcquisitionFilePath == "" {
return fmt.Errorf("no acquisition_path nor acquisition_dir")
}
c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
c.Crowdsec.DataDir = c.ConfigPaths.DataDir
c.Crowdsec.HubDir = c.ConfigPaths.HubDir
c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
if c.Crowdsec.ParserRoutinesCount <= 0 {
c.Crowdsec.ParserRoutinesCount = 1
}
if c.Crowdsec.BucketsRoutinesCount <= 0 {
c.Crowdsec.BucketsRoutinesCount = 1
}
if c.Crowdsec.OutputRoutinesCount <= 0 {
c.Crowdsec.OutputRoutinesCount = 1
}
}
if err := c.CleanupPaths(); err != nil {
return errors.Wrap(err, "invalid config")
}
if c.Cscli != nil {
c.Cscli.DbConfig = c.DbConfig
c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
c.Cscli.DataDir = c.ConfigPaths.DataDir
c.Cscli.HubDir = c.ConfigPaths.HubDir
c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
if c.Cscli.PrometheusUrl == "" {
port := 6060
if c.Prometheus.ListenPort != 0 {
port = c.Prometheus.ListenPort
}
c.Cscli.PrometheusUrl = fmt.Sprintf("http://127.0.0.1:%d/", port)
}
}
if c.API.Client != nil && c.API.Client.CredentialsFilePath != "" && !c.DisableAgent {
fcontent, err := ioutil.ReadFile(c.API.Client.CredentialsFilePath)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to read api client credential configuration file '%s'", c.API.Client.CredentialsFilePath))
}
err = yaml.UnmarshalStrict(fcontent, &c.API.Client.Credentials)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed unmarshaling api client credential configuration file '%s'", c.API.Client.CredentialsFilePath))
}
if c.API.Client.Credentials != nil && c.API.Client.Credentials.URL != "" {
if !strings.HasSuffix(c.API.Client.Credentials.URL, "/") {
c.API.Client.Credentials.URL = c.API.Client.Credentials.URL + "/"
}
}
if c.API.Client.InsecureSkipVerify == nil {
apiclient.InsecureSkipVerify = false
} else {
apiclient.InsecureSkipVerify = *c.API.Client.InsecureSkipVerify
}
}
if c.API.Server != nil && !c.DisableAPI {
c.API.Server.DbConfig = c.DbConfig
c.API.Server.LogDir = c.Common.LogDir
c.API.Server.LogMedia = c.Common.LogMedia
if err := c.API.Server.LoadProfiles(); err != nil {
return errors.Wrap(err, "while loading profiles for LAPI")
}
if c.API.Server.OnlineClient != nil && c.API.Server.OnlineClient.CredentialsFilePath != "" {
c.API.Server.OnlineClient.Credentials = new(ApiCredentialsCfg)
fcontent, err := ioutil.ReadFile(c.API.Server.OnlineClient.CredentialsFilePath)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to read api server credentials configuration file '%s'", c.API.Server.OnlineClient.CredentialsFilePath))
}
err = yaml.UnmarshalStrict(fcontent, c.API.Server.OnlineClient.Credentials)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed unmarshaling api server credentials configuration file '%s'", c.API.Server.OnlineClient.CredentialsFilePath))
}
if c.API.Server.OnlineClient.Credentials.Login == "" || c.API.Server.OnlineClient.Credentials.Password == "" || c.API.Server.OnlineClient.Credentials.URL == "" {
log.Debugf("can't load CAPI credentials from '%s' (missing field)", c.API.Server.OnlineClient.CredentialsFilePath)
c.API.Server.OnlineClient.Credentials = nil
}
}
if c.API.Server.OnlineClient == nil || c.API.Server.OnlineClient.Credentials == nil {
log.Printf("push and pull to crowdsec API disabled")
}
}
return nil
}
func (c *GlobalConfig) LoadSimulation() error {
if c.ConfigPaths == nil {
return fmt.Errorf("ConfigPaths is empty")
}
simCfg := SimulationConfig{}
if c.ConfigPaths.SimulationFilePath == "" {
c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")
}
rcfg, err := ioutil.ReadFile(c.ConfigPaths.SimulationFilePath)
if err != nil {
return errors.Wrapf(err, "while reading '%s'", c.ConfigPaths.SimulationFilePath)
} else {
if err := yaml.UnmarshalStrict(rcfg, &simCfg); err != nil {
return fmt.Errorf("while unmarshaling simulation file '%s' : %s", c.ConfigPaths.SimulationFilePath, err)
}
}
if simCfg.Simulation == nil {
simCfg.Simulation = new(bool)
}
if c.Crowdsec != nil {
c.Crowdsec.SimulationConfig = &simCfg
}
if c.Cscli != nil {
c.Cscli.SimulationConfig = &simCfg
}
return nil
}
func NewConfig() *GlobalConfig {
cfg := GlobalConfig{}
return &cfg
}
func NewDefaultConfig() *GlobalConfig {
func NewDefaultConfig() *Config {
logLevel := log.InfoLevel
CommonCfg := CommonCfg{
Daemonize: false,
@ -270,7 +103,7 @@ func NewDefaultConfig() *GlobalConfig {
DbPath: "/var/lib/crowdsec/data/crowdsec.db",
}
globalCfg := GlobalConfig{
globalCfg := Config{
Common: &CommonCfg,
Prometheus: &prometheus,
Crowdsec: &crowdsecCfg,
@ -282,60 +115,3 @@ func NewDefaultConfig() *GlobalConfig {
return &globalCfg
}
func (c *GlobalConfig) CleanupPaths() error {
var err error
if c.Common != nil {
var CommonCleanup = []*string{
&c.Common.PidDir,
&c.Common.LogDir,
&c.Common.WorkingDir,
}
for _, k := range CommonCleanup {
if *k == "" {
continue
}
*k, err = filepath.Abs(*k)
if err != nil {
return errors.Wrap(err, "failed to clean path")
}
}
}
if c.Crowdsec != nil {
var crowdsecCleanup = []*string{
&c.Crowdsec.AcquisitionFilePath,
}
for _, k := range crowdsecCleanup {
if *k == "" {
continue
}
*k, err = filepath.Abs(*k)
if err != nil {
return errors.Wrap(err, "failed to clean path")
}
}
}
if c.ConfigPaths != nil {
var configPathsCleanup = []*string{
&c.ConfigPaths.HubDir,
&c.ConfigPaths.HubIndexFile,
&c.ConfigPaths.ConfigDir,
&c.ConfigPaths.DataDir,
&c.ConfigPaths.SimulationFilePath,
}
for _, k := range configPathsCleanup {
if *k == "" {
continue
}
*k, err = filepath.Abs(*k)
if err != nil {
return errors.Wrap(err, "failed to clean path")
}
}
}
return nil
}

45
pkg/csconfig/config_paths.go
View file

@ -1,5 +1,12 @@
package csconfig
import (
"fmt"
"path/filepath"
"github.com/pkg/errors"
)
type ConfigurationPaths struct {
ConfigDir string `yaml:"config_dir"`
DataDir string `yaml:"data_dir,omitempty"`
@ -7,3 +14,41 @@ type ConfigurationPaths struct {
HubIndexFile string `yaml:"index_path,omitempty"` //path of the .index.json
HubDir string `yaml:"hub_dir,omitempty"`
}
func (c *Config) LoadConfigurationPaths() error {
var err error
if c.ConfigPaths == nil {
return fmt.Errorf("no configuration paths provided")
}
if c.ConfigPaths.DataDir == "" {
return fmt.Errorf("please provide a data directory with the 'data_dir' directive in the 'config_paths' section")
}
if c.ConfigPaths.HubDir == "" {
c.ConfigPaths.HubDir = filepath.Clean(c.ConfigPaths.ConfigDir + "/hub")
}
if c.ConfigPaths.HubIndexFile == "" {
c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json")
}
var configPathsCleanup = []*string{
&c.ConfigPaths.HubDir,
&c.ConfigPaths.HubIndexFile,
&c.ConfigPaths.ConfigDir,
&c.ConfigPaths.DataDir,
&c.ConfigPaths.SimulationFilePath,
}
for _, k := range configPathsCleanup {
if *k == "" {
continue
}
*k, err = filepath.Abs(*k)
if err != nil {
return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
}
}
return nil
}

180
pkg/csconfig/config_test.go
View file

@ -2,203 +2,59 @@ package csconfig
import (
"fmt"
"log"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
)
func TestDefaultConfig(t *testing.T) {
x := NewDefaultConfig()
x.Dump()
}
func TestNormalLoad(t *testing.T) {
x := NewConfig()
err := x.LoadConfigurationFile("./tests/config.yaml", false, false)
_, err := NewConfig("./tests/config.yaml", false, false)
if err != nil {
t.Fatalf("unexpected error %s", err)
}
x = NewConfig()
err = x.LoadConfigurationFile("./tests/xxx.yaml", false, false)
_, err = NewConfig("./tests/xxx.yaml", false, false)
if fmt.Sprintf("%s", err) != "failed to read config file: open ./tests/xxx.yaml: no such file or directory" {
t.Fatalf("unexpected error %s", err)
}
x = NewConfig()
err = x.LoadConfigurationFile("./tests/simulation.yaml", false, false)
if !strings.HasPrefix(fmt.Sprintf("%s", err), "failed unmarshaling config: yaml: unmarshal error") {
_, err = NewConfig("./tests/simulation.yaml", false, false)
if !strings.HasPrefix(fmt.Sprintf("%s", err), "yaml: unmarshal errors:") {
t.Fatalf("unexpected error %s", err)
}
}
func TestCleanupPaths(t *testing.T) {
tests := []struct {
name string
Input *GlobalConfig
expectedResult *GlobalConfig
err string
}{
{
name: "daemon cleanup",
Input: &GlobalConfig{
Common: &CommonCfg{
PidDir: "////tmp//",
LogDir: "/////tmp///",
WorkingDir: "/////tmp///",
},
},
expectedResult: &GlobalConfig{
Common: &CommonCfg{
PidDir: "/tmp",
LogDir: "/tmp",
WorkingDir: "/tmp",
},
},
},
//
{
name: "crowdsec cleanup",
Input: &GlobalConfig{
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "////tmp//x.yaml",
},
},
expectedResult: &GlobalConfig{
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "/tmp/x.yaml",
},
},
},
//
{
name: "config paths cleanup",
Input: &GlobalConfig{
ConfigPaths: &ConfigurationPaths{
HubDir: "////tmp//",
HubIndexFile: "////tmp//x.yaml",
ConfigDir: "////tmp//",
DataDir: "////tmp//",
SimulationFilePath: "//tmp///toto.yaml",
},
},
expectedResult: &GlobalConfig{
ConfigPaths: &ConfigurationPaths{
HubDir: "/tmp",
HubIndexFile: "/tmp/x.yaml",
ConfigDir: "/tmp",
DataDir: "/tmp",
SimulationFilePath: "/tmp/toto.yaml",
},
},
},
}
for idx, test := range tests {
err := test.Input.CleanupPaths()
if test.err != "" {
if strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
t.Fatalf("%d/%d expected err %s got %s", idx, len(tests), test.err, fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input)
if !isOk {
t.Fatalf("%d/%d failed test", idx, len(tests))
}
}
}
func TestSimulationLoading(t *testing.T) {
tests := []struct {
name string
Input *GlobalConfig
expectedResult *SimulationConfig
err string
}{
{
name: "basic valid simulation",
Input: &GlobalConfig{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "./tests/simulation.yaml",
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedResult: &SimulationConfig{Simulation: new(bool)},
},
{
name: "basic bad file name",
Input: &GlobalConfig{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "./tests/xxx.yaml",
},
Crowdsec: &CrowdsecServiceCfg{},
},
err: "while reading './tests/xxx.yaml': open ./tests/xxx.yaml: no such file or directory",
},
{
name: "basic nil config",
Input: &GlobalConfig{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "",
},
Crowdsec: &CrowdsecServiceCfg{},
},
},
{
name: "basic bad file content",
Input: &GlobalConfig{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "./tests/config.yaml",
},
Crowdsec: &CrowdsecServiceCfg{},
},
err: "while unmarshaling simulation file './tests/config.yaml' : yaml: unmarshal errors",
},
}
for idx, test := range tests {
err := test.Input.LoadSimulation()
if err == nil && test.err != "" {
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Crowdsec.SimulationConfig)
if !isOk {
t.Fatalf("test '%s' failed", test.name)
}
}
}
func TestNewCrowdSecConfig(t *testing.T) {
tests := []struct {
name string
expectedResult *GlobalConfig
expectedResult *Config
err string
}{
{
name: "new configuration: basic",
expectedResult: &GlobalConfig{},
expectedResult: &Config{},
err: "",
},
}
for _, test := range tests {
result := NewConfig()
result := &Config{}
isOk := assert.Equal(t, test.expectedResult, result)
if !isOk {
t.Fatalf("test '%s' failed", test.name)
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
log.Infof("test '%s' : OK", test.name)
}
}
func TestDefaultConfig(t *testing.T) {
x := NewDefaultConfig()
if err := x.Dump(); err != nil {
log.Fatal(err)
}
}

88
pkg/csconfig/crowdsec_service.go
View file

@ -1,5 +1,14 @@
package csconfig
import (
"fmt"
"os"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
/*Configurations needed for crowdsec to load parser/scenarios/... + acquisition*/
type CrowdsecServiceCfg struct {
AcquisitionFilePath string `yaml:"acquisition_path,omitempty"`
@ -21,3 +30,82 @@ type CrowdsecServiceCfg struct {
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
}
func (c *Config) LoadCrowdsec() error {
var err error
// Configuration paths are dependency to load crowdsec configuration
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
if c.Crowdsec == nil {
log.Warningf("crowdsec agent is disabled")
c.DisableAgent = true
return nil
}
if c.Crowdsec.AcquisitionFilePath != "" {
log.Debugf("non-empty acquisition file path %s", c.Crowdsec.AcquisitionFilePath)
if _, err := os.Stat(c.Crowdsec.AcquisitionFilePath); err != nil {
return errors.Wrapf(err, "while checking acquisition path %s", c.Crowdsec.AcquisitionFilePath)
}
c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, c.Crowdsec.AcquisitionFilePath)
}
if c.Crowdsec.AcquisitionDirPath != "" {
c.Crowdsec.AcquisitionDirPath, err = filepath.Abs(c.Crowdsec.AcquisitionDirPath)
if err != nil {
return errors.Wrapf(err, "can't get absolute path of '%s'", c.Crowdsec.AcquisitionDirPath)
}
files, err := filepath.Glob(c.Crowdsec.AcquisitionDirPath + "/*.yaml")
if err != nil {
return errors.Wrap(err, "while globing acquis_dir")
}
c.Crowdsec.AcquisitionFiles = append(c.Crowdsec.AcquisitionFiles, files...)
}
if c.Crowdsec.AcquisitionDirPath == "" && c.Crowdsec.AcquisitionFilePath == "" {
return fmt.Errorf("no acquisition_path nor acquisition_dir")
}
c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
c.Crowdsec.DataDir = c.ConfigPaths.DataDir
c.Crowdsec.HubDir = c.ConfigPaths.HubDir
c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
if c.Crowdsec.ParserRoutinesCount <= 0 {
c.Crowdsec.ParserRoutinesCount = 1
}
if c.Crowdsec.BucketsRoutinesCount <= 0 {
c.Crowdsec.BucketsRoutinesCount = 1
}
if c.Crowdsec.OutputRoutinesCount <= 0 {
c.Crowdsec.OutputRoutinesCount = 1
}
var crowdsecCleanup = []*string{
&c.Crowdsec.AcquisitionFilePath,
}
for _, k := range crowdsecCleanup {
if *k == "" {
continue
}
*k, err = filepath.Abs(*k)
if err != nil {
return errors.Wrapf(err, "failed to get absolute path of '%s'", *k)
}
}
for i, file := range c.Crowdsec.AcquisitionFiles {
f, err := filepath.Abs(file)
if err != nil {
return errors.Wrapf(err, "failed to get absolute path of '%s'", file)
}
c.Crowdsec.AcquisitionFiles[i] = f
}
if err := c.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %s", err.Error())
}
if err := c.LoadHub(); err != nil {
return fmt.Errorf("loading hub: %s", err)
}
return nil
}

192
pkg/csconfig/crowdsec_service_test.go Normal file
View file

@ -0,0 +1,192 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadCrowdsec(t *testing.T) {
acquisFullPath, err := filepath.Abs("./tests/acquis.yaml")
if err != nil {
t.Fatalf(err.Error())
}
acquisInDirFullPath, err := filepath.Abs("./tests/acquis/acquis.yaml")
if err != nil {
t.Fatalf(err.Error())
}
acquisDirFullPath, err := filepath.Abs("./tests/acquis")
if err != nil {
t.Fatalf(err.Error())
}
hubFullPath, err := filepath.Abs("./hub")
if err != nil {
t.Fatalf(err.Error())
}
dataFullPath, err := filepath.Abs("./data")
if err != nil {
t.Fatalf(err.Error())
}
configDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatalf(err.Error())
}
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
if err != nil {
t.Fatalf(err.Error())
}
tests := []struct {
name string
Input *Config
expectedResult *CrowdsecServiceCfg
err string
}{
{
name: "basic valid configuration",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "./tests/acquis.yaml",
},
},
expectedResult: &CrowdsecServiceCfg{
AcquisitionDirPath: "",
AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
AcquisitionFiles: []string{acquisFullPath},
},
},
{
name: "basic valid configuration with acquisition dir",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "./tests/acquis.yaml",
AcquisitionDirPath: "./tests/acquis/",
},
},
expectedResult: &CrowdsecServiceCfg{
AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath,
ConfigDir: configDirFullPath,
HubIndexFile: hubIndexFileFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath},
},
},
{
name: "no acquisition file and dir",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{},
},
expectedResult: &CrowdsecServiceCfg{
BucketsRoutinesCount: 0,
ParserRoutinesCount: 0,
OutputRoutinesCount: 0,
},
},
{
name: "non existing acquisition file",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
DataDir: "./data",
HubDir: "./hub",
},
API: &APICfg{
Client: &LocalApiClientCfg{
CredentialsFilePath: "./tests/lapi-secrets.yaml",
},
},
Crowdsec: &CrowdsecServiceCfg{
AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
},
},
expectedResult: &CrowdsecServiceCfg{
AcquisitionFilePath: "./tests/acquis_not_exist.yaml",
BucketsRoutinesCount: 0,
ParserRoutinesCount: 0,
OutputRoutinesCount: 0,
},
},
{
name: "agent disabled",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
DataDir: "./data",
HubDir: "./hub",
},
},
expectedResult: nil,
},
}
for idx, test := range tests {
fmt.Printf("TEST '%s'\n", test.name)
err := test.Input.LoadCrowdsec()
if err == nil && test.err != "" {
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Crowdsec)
if !isOk {
t.Fatalf("test '%s' failed", test.name)
}
}
}

15
pkg/csconfig/cscli.go
View file

@ -13,3 +13,18 @@ type CscliCfg struct {
SimulationFilePath string `yaml:"-"`
PrometheusUrl string `yaml:"prometheus_uri"`
}
func (c *Config) LoadCSCLI() error {
if c.Cscli == nil {
c.Cscli = &CscliCfg{}
}
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
c.Cscli.DataDir = c.ConfigPaths.DataDir
c.Cscli.HubDir = c.ConfigPaths.HubDir
c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
return nil
}

84
pkg/csconfig/cscli_test.go Normal file
View file

@ -0,0 +1,84 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadCSCLI(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
if err != nil {
t.Fatalf(err.Error())
}
dataFullPath, err := filepath.Abs("./data")
if err != nil {
t.Fatalf(err.Error())
}
configDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatalf(err.Error())
}
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
if err != nil {
t.Fatalf(err.Error())
}
tests := []struct {
name string
Input *Config
expectedResult *CscliCfg
err string
}{
{
name: "basic valid configuration",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
DataDir: "./data",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
},
expectedResult: &CscliCfg{
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
},
},
{
name: "no configuration path",
Input: &Config{},
expectedResult: &CscliCfg{},
},
}
for idx, test := range tests {
err := test.Input.LoadCSCLI()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}

22
pkg/csconfig/database.go
View file

@ -1,6 +1,10 @@
package csconfig
import log "github.com/sirupsen/logrus"
import (
"fmt"
log "github.com/sirupsen/logrus"
)
type DatabaseCfg struct {
User string `yaml:"user"`
@ -18,3 +22,19 @@ type FlushDBCfg struct {
MaxItems *int `yaml:"max_items"`
MaxAge *string `yaml:"max_age"`
}
func (c *Config) LoadDBConfig() error {
if c.DbConfig == nil {
return fmt.Errorf("no database configuration provided")
}
if c.Cscli != nil {
c.Cscli.DbConfig = c.DbConfig
}
if c.API != nil && c.API.Server != nil {
c.API.Server.DbConfig = c.DbConfig
}
return nil
}

62
pkg/csconfig/database_test.go Normal file
View file

@ -0,0 +1,62 @@
package csconfig
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadDBConfig(t *testing.T) {
tests := []struct {
name string
Input *Config
expectedResult *DatabaseCfg
err string
}{
{
name: "basic valid configuration",
Input: &Config{
DbConfig: &DatabaseCfg{
Type: "sqlite",
DbPath: "./tests/test.db",
},
Cscli: &CscliCfg{},
API: &APICfg{
Server: &LocalApiServerCfg{},
},
},
expectedResult: &DatabaseCfg{
Type: "sqlite",
DbPath: "./tests/test.db",
},
},
{
name: "no configuration path",
Input: &Config{},
expectedResult: nil,
},
}
for idx, test := range tests {
err := test.Input.LoadDBConfig()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.DbConfig)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}

24
pkg/csconfig/hub.go Normal file
View file

@ -0,0 +1,24 @@
package csconfig
/*cscli specific config, such as hub directory*/
type Hub struct {
HubDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
DataDir string `yaml:"-"`
}
func (c *Config) LoadHub() error {
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
c.Hub = &Hub{
HubIndexFile: c.ConfigPaths.HubIndexFile,
ConfigDir: c.ConfigPaths.ConfigDir,
HubDir: c.ConfigPaths.HubDir,
DataDir: c.ConfigPaths.DataDir,
}
return nil
}

94
pkg/csconfig/hub_test.go Normal file
View file

@ -0,0 +1,94 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadHub(t *testing.T) {
hubFullPath, err := filepath.Abs("./hub")
if err != nil {
t.Fatalf(err.Error())
}
dataFullPath, err := filepath.Abs("./data")
if err != nil {
t.Fatalf(err.Error())
}
configDirFullPath, err := filepath.Abs("./tests")
if err != nil {
t.Fatalf(err.Error())
}
hubIndexFileFullPath, err := filepath.Abs("./hub/.index.json")
if err != nil {
t.Fatalf(err.Error())
}
tests := []struct {
name string
Input *Config
expectedResult *Hub
err string
}{
{
name: "basic valid configuration",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
DataDir: "./data",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
},
expectedResult: &Hub{
ConfigDir: configDirFullPath,
DataDir: dataFullPath,
HubDir: hubFullPath,
HubIndexFile: hubIndexFileFullPath,
},
},
{
name: "no data dir",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
ConfigDir: "./tests",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
},
},
expectedResult: nil,
},
{
name: "no configuration path",
Input: &Config{},
expectedResult: nil,
},
}
for idx, test := range tests {
err := test.Input.LoadHub()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Hub)
if !isOk {
t.Fatalf("TEST '%s': NOK", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}

12
pkg/csconfig/prometheus.go
View file

@ -1,5 +1,7 @@
package csconfig
import "fmt"
/**/
type PrometheusCfg struct {
Enabled bool `yaml:"enabled"`
@ -7,3 +9,13 @@ type PrometheusCfg struct {
ListenAddr string `yaml:"listen_addr"`
ListenPort int `yaml:"listen_port"`
}
func (c *Config) LoadPrometheus() error {
if c.Cscli != nil && c.Cscli.PrometheusUrl == "" && c.Prometheus != nil {
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)
}
}
return nil
}

55
pkg/csconfig/prometheus_test.go Normal file
View file

@ -0,0 +1,55 @@
package csconfig
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadPrometheus(t *testing.T) {
tests := []struct {
name string
Input *Config
expectedResult string
err string
}{
{
name: "basic valid configuration",
Input: &Config{
Prometheus: &PrometheusCfg{
Enabled: true,
Level: "full",
ListenAddr: "127.0.0.1",
ListenPort: 6060,
},
Cscli: &CscliCfg{},
},
expectedResult: "http://127.0.0.1:6060",
},
}
for idx, test := range tests {
err := test.Input.LoadPrometheus()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli.PrometheusUrl)
if !isOk {
t.Fatalf("test '%s' failed\n", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}

39
pkg/csconfig/simulation.go
View file

@ -1,5 +1,14 @@
package csconfig
import (
"fmt"
"io/ioutil"
"path/filepath"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
type SimulationConfig struct {
Simulation *bool `yaml:"simulation"`
Exclusions []string `yaml:"exclusions,omitempty"`
@ -19,3 +28,33 @@ func (s *SimulationConfig) IsSimulated(scenario string) bool {
}
return simulated
}
func (c *Config) LoadSimulation() error {
if err := c.LoadConfigurationPaths(); err != nil {
return err
}
simCfg := SimulationConfig{}
if c.ConfigPaths.SimulationFilePath == "" {
c.ConfigPaths.SimulationFilePath = filepath.Clean(c.ConfigPaths.ConfigDir + "/simulation.yaml")
}
rcfg, err := ioutil.ReadFile(c.ConfigPaths.SimulationFilePath)
if err != nil {
return errors.Wrapf(err, "while reading '%s'", c.ConfigPaths.SimulationFilePath)
} else {
if err := yaml.UnmarshalStrict(rcfg, &simCfg); err != nil {
return fmt.Errorf("while unmarshaling simulation file '%s' : %s", c.ConfigPaths.SimulationFilePath, err)
}
}
if simCfg.Simulation == nil {
simCfg.Simulation = new(bool)
}
if c.Crowdsec != nil {
c.Crowdsec.SimulationConfig = &simCfg
}
if c.Cscli != nil {
c.Cscli.SimulationConfig = &simCfg
}
return nil
}

156
pkg/csconfig/simulation_test.go Normal file
View file

@ -0,0 +1,156 @@
package csconfig
import (
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSimulationLoading(t *testing.T) {
testXXFullPath, err := filepath.Abs("./tests/xxx.yaml")
if err != nil {
panic(err)
}
badYamlFullPath, err := filepath.Abs("./tests/config.yaml")
if err != nil {
panic(err)
}
tests := []struct {
name string
Input *Config
expectedResult *SimulationConfig
err string
}{
{
name: "basic valid simulation",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "./tests/simulation.yaml",
DataDir: "./data",
},
Crowdsec: &CrowdsecServiceCfg{},
Cscli: &CscliCfg{},
},
expectedResult: &SimulationConfig{Simulation: new(bool)},
},
{
name: "basic bad file name",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "./tests/xxx.yaml",
DataDir: "./data",
},
Crowdsec: &CrowdsecServiceCfg{},
},
err: fmt.Sprintf("while reading '%s': open %s: no such file or directory", testXXFullPath, testXXFullPath),
},
{
name: "basic nil config",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "",
DataDir: "./data",
},
Crowdsec: &CrowdsecServiceCfg{},
},
},
{
name: "basic bad file content",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "./tests/config.yaml",
DataDir: "./data",
},
Crowdsec: &CrowdsecServiceCfg{},
},
err: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
},
{
name: "basic bad file content",
Input: &Config{
ConfigPaths: &ConfigurationPaths{
SimulationFilePath: "./tests/config.yaml",
DataDir: "./data",
},
Crowdsec: &CrowdsecServiceCfg{},
},
err: fmt.Sprintf("while unmarshaling simulation file '%s' : yaml: unmarshal errors", badYamlFullPath),
},
}
for idx, test := range tests {
err := test.Input.LoadSimulation()
if err == nil && test.err != "" {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests))
} else if test.err != "" {
if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) {
fmt.Printf("TEST '%s': NOK\n", test.name)
t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests),
test.err,
fmt.Sprintf("%s", err))
}
}
isOk := assert.Equal(t, test.expectedResult, test.Input.Crowdsec.SimulationConfig)
if !isOk {
t.Fatalf("TEST '%s': NOK\n", test.name)
} else {
fmt.Printf("TEST '%s': OK\n", test.name)
}
}
}
func TestIsSimulated(t *testing.T) {
simCfgOff := &SimulationConfig{
Simulation: new(bool),
Exclusions: []string{"test"},
}
simCfgOn := &SimulationConfig{
Simulation: new(bool),
Exclusions: []string{"test"},
}
*simCfgOn.Simulation = true
tests := []struct {
name string
SimulationConfig *SimulationConfig
Input string
expectedResult bool
err string
}{
{
name: "No simulation except (in exclusion)",
SimulationConfig: simCfgOff,
Input: "test",
expectedResult: true,
},
{
name: "All simulation (not in exclusion)",
SimulationConfig: simCfgOn,
Input: "toto",
expectedResult: true,
},
{
name: "All simulation (in exclusion)",
SimulationConfig: simCfgOn,
Input: "test",
expectedResult: false,
},
}
for _, test := range tests {
IsSimulated := test.SimulationConfig.IsSimulated(test.Input)
isOk := assert.Equal(t, test.expectedResult, IsSimulated)
if !isOk {
fmt.Printf("TEST: '%v' failed", test.name)
t.Fatal()
}
}
}

0
pkg/csconfig/tests/acquis/acquis.yaml Normal file
View file

1
pkg/csconfig/tests/bad_lapi-secrets.yaml Normal file
View file

@ -0,0 +1 @@
unknown_key: test

3
pkg/csconfig/tests/bad_online-api-secrets.yaml Normal file
View file

@ -0,0 +1,3 @@
login: test
password:
url:

3
pkg/csconfig/tests/lapi-secrets.yaml
View file

@ -0,0 +1,3 @@
url: http://localhost:8080
login: test
password: testpassword

3
pkg/csconfig/tests/online-api-secrets.yaml
View file

@ -0,0 +1,3 @@
url: http://crowdsec.api
login: test
password: testpassword

76
pkg/cwhub/cwhub_test.go
View file

@ -27,12 +27,12 @@ var testDataFolder = "."
func TestItemStatus(t *testing.T) {
cfg := test_prepenv()
err := UpdateHubIdx(cfg.Cscli)
err := UpdateHubIdx(cfg.Hub)
//DownloadHubIdx()
if err != nil {
t.Fatalf("failed to download index : %s", err)
}
if err := GetHubIdx(cfg.Cscli); err != nil {
if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index : %s", err)
}
@ -74,12 +74,12 @@ func TestItemStatus(t *testing.T) {
func TestGetters(t *testing.T) {
cfg := test_prepenv()
err := UpdateHubIdx(cfg.Cscli)
err := UpdateHubIdx(cfg.Hub)
//DownloadHubIdx()
if err != nil {
t.Fatalf("failed to download index : %s", err)
}
if err := GetHubIdx(cfg.Cscli); err != nil {
if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index : %s", err)
}
@ -135,58 +135,58 @@ func TestIndexDownload(t *testing.T) {
cfg := test_prepenv()
err := UpdateHubIdx(cfg.Cscli)
err := UpdateHubIdx(cfg.Hub)
//DownloadHubIdx()
if err != nil {
t.Fatalf("failed to download index : %s", err)
}
if err := GetHubIdx(cfg.Cscli); err != nil {
if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index : %s", err)
}
}
func test_prepenv() *csconfig.GlobalConfig {
func test_prepenv() *csconfig.Config {
log.SetLevel(log.DebugLevel)
var cfg = csconfig.NewConfig()
cfg.Cscli = &csconfig.CscliCfg{}
cfg.Cscli.ConfigDir, _ = filepath.Abs("./install")
cfg.Cscli.HubDir, _ = filepath.Abs("./hubdir")
cfg.Cscli.HubIndexFile = filepath.Clean("./hubdir/.index.json")
var cfg = &csconfig.Config{}
cfg.Hub = &csconfig.Hub{}
cfg.Hub.ConfigDir, _ = filepath.Abs("./install")
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
//Mock the http client
http.DefaultClient.Transport = newMockTransport()
if err := os.RemoveAll(cfg.Cscli.ConfigDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Cscli.ConfigDir, err)
if err := os.RemoveAll(cfg.Hub.ConfigDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.ConfigDir, err)
}
if err := os.MkdirAll(cfg.Cscli.ConfigDir, 0700); err != nil {
if err := os.MkdirAll(cfg.Hub.ConfigDir, 0700); err != nil {
log.Fatalf("mkdir : %s", err)
}
if err := os.RemoveAll(cfg.Cscli.HubDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Cscli.HubDir, err)
if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
}
if err := os.MkdirAll(cfg.Cscli.HubDir, 0700); err != nil {
log.Fatalf("failed to mkdir %s : %s", cfg.Cscli.HubDir, err)
if err := os.MkdirAll(cfg.Hub.HubDir, 0700); err != nil {
log.Fatalf("failed to mkdir %s : %s", cfg.Hub.HubDir, err)
}
if err := UpdateHubIdx(cfg.Cscli); err != nil {
if err := UpdateHubIdx(cfg.Hub); err != nil {
log.Fatalf("failed to download index : %s", err)
}
// if err := os.RemoveAll(cfg.Cscli.InstallDir); err != nil {
// log.Fatalf("failed to remove %s : %s", cfg.Cscli.InstallDir, err)
// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
// log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
// }
// if err := os.MkdirAll(cfg.Cscli.InstallDir, 0700); err != nil {
// log.Fatalf("failed to mkdir %s : %s", cfg.Cscli.InstallDir, err)
// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
// log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
// }
return cfg
}
func testInstallItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
//Install the parser
item, err := DownloadLatest(cfg, item, false)
@ -218,7 +218,7 @@ func testInstallItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
}
}
func testTaintItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if hubIdx[item.Type][item.Name].Tainted {
t.Fatalf("pre-taint: %s should not be tainted", item.Name)
}
@ -240,7 +240,7 @@ func testTaintItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
}
}
func testUpdateItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
func testUpdateItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if hubIdx[item.Type][item.Name].UpToDate {
t.Fatalf("update: %s should NOT be up-to-date", item.Name)
@ -262,7 +262,7 @@ func testUpdateItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
}
}
func testDisableItem(cfg *csconfig.CscliCfg, t *testing.T, item Item) {
func testDisableItem(cfg *csconfig.Hub, t *testing.T, item Item) {
if !item.Installed {
t.Fatalf("disable: %s should be installed", item.Name)
}
@ -314,20 +314,20 @@ func TestInstallParser(t *testing.T) {
*/
cfg := test_prepenv()
if err := GetHubIdx(cfg.Cscli); err != nil {
if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index")
}
//map iteration is random by itself
for _, it := range hubIdx[PARSERS] {
testInstallItem(cfg.Cscli, t, it)
testInstallItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = HubStatus(PARSERS, it.Name, false)
testTaintItem(cfg.Cscli, t, it)
testTaintItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
_ = HubStatus(PARSERS, it.Name, false)
testUpdateItem(cfg.Cscli, t, it)
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
testDisableItem(cfg.Cscli, t, it)
testDisableItem(cfg.Hub, t, it)
it = hubIdx[PARSERS][it.Name]
break
@ -347,18 +347,18 @@ func TestInstallCollection(t *testing.T) {
*/
cfg := test_prepenv()
if err := GetHubIdx(cfg.Cscli); err != nil {
if err := GetHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to load hub index")
}
//map iteration is random by itself
for _, it := range hubIdx[COLLECTIONS] {
testInstallItem(cfg.Cscli, t, it)
testInstallItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testTaintItem(cfg.Cscli, t, it)
testTaintItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testUpdateItem(cfg.Cscli, t, it)
testUpdateItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
testDisableItem(cfg.Cscli, t, it)
testDisableItem(cfg.Hub, t, it)
it = hubIdx[COLLECTIONS][it.Name]
x := HubStatus(COLLECTIONS, it.Name, false)

30
pkg/cwhub/download.go
View file

@ -22,9 +22,9 @@ import (
"gopkg.in/yaml.v2"
)
func UpdateHubIdx(cscli *csconfig.CscliCfg) error {
func UpdateHubIdx(hub *csconfig.Hub) error {
bidx, err := DownloadHubIdx(cscli)
bidx, err := DownloadHubIdx(hub)
if err != nil {
return errors.Wrap(err, "failed to download index")
}
@ -35,13 +35,13 @@ func UpdateHubIdx(cscli *csconfig.CscliCfg) error {
}
}
hubIdx = ret
if err := LocalSync(cscli); err != nil {
if err := LocalSync(hub); err != nil {
return errors.Wrap(err, "failed to sync")
}
return nil
}
func DownloadHubIdx(cscli *csconfig.CscliCfg) ([]byte, error) {
func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
if err != nil {
@ -59,7 +59,7 @@ func DownloadHubIdx(cscli *csconfig.CscliCfg) ([]byte, error) {
if err != nil {
return nil, errors.Wrap(err, "failed to read request answer for hub index")
}
file, err := os.OpenFile(cscli.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, errors.Wrap(err, "while opening hub index file")
@ -70,12 +70,12 @@ func DownloadHubIdx(cscli *csconfig.CscliCfg) ([]byte, error) {
if err != nil {
return nil, errors.Wrap(err, "while writting hub index file")
}
log.Infof("Wrote new %d bytes index to %s", wsize, cscli.HubIndexFile)
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
return body, nil
}
//DownloadLatest will download the latest version of Item to the tdir directory
func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item, error) {
func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
@ -89,12 +89,12 @@ func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item
//recurse as it's a collection
if ptrtype == COLLECTIONS {
log.Tracef("collection, recurse")
hubIdx[ptrtype][p], err = DownloadLatest(cscli, val, overwrite)
hubIdx[ptrtype][p], err = DownloadLatest(hub, val, overwrite)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
}
}
item, err := DownloadItem(cscli, val, overwrite)
item, err := DownloadItem(hub, val, overwrite)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", val.Name))
}
@ -102,7 +102,7 @@ func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !item.Installed && !val.Downloaded {
if item, err = EnableItem(cscli, item); err != nil {
if item, err = EnableItem(hub, item); err != nil {
return target, errors.Wrapf(err, "enabling '%s'", item.Name)
}
}
@ -112,20 +112,20 @@ func DownloadLatest(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item
}
}
}
target, err = DownloadItem(cscli, target, overwrite)
target, err = DownloadItem(hub, target, overwrite)
if err != nil {
return target, fmt.Errorf("failed to download item : %s", err)
}
} else {
return DownloadItem(cscli, target, overwrite)
return DownloadItem(hub, target, overwrite)
}
return target, nil
}
func DownloadItem(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item, error) {
func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error) {
var tdir = cscli.HubDir
var dataFolder = cscli.DataDir
var tdir = hub.HubDir
var dataFolder = hub.DataDir
/*if user didn't --force, don't overwrite local, tainted, up-to-date files*/
if !overwrite {
if target.Tainted {

12
pkg/cwhub/download_test.go
View file

@ -12,21 +12,27 @@ import (
func TestDownloadHubIdx(t *testing.T) {
back := RawFileURLTemplate
//bad url template
fmt.Println("Test 'bad URL'")
RawFileURLTemplate = "x"
ret, err := DownloadHubIdx(&csconfig.CscliCfg{})
ret, err := DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
log.Errorf("unexpected error %s", err)
}
fmt.Printf("->%+v", ret)
//bad domain
fmt.Println("Test 'bad domain'")
RawFileURLTemplate = "https://baddomain/crowdsecurity/hub/%s/%s"
ret, err = DownloadHubIdx(&csconfig.CscliCfg{})
ret, err = DownloadHubIdx(&csconfig.Hub{})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
log.Errorf("unexpected error %s", err)
}
fmt.Printf("->%+v", ret)
//bad target path
fmt.Println("Test 'bad target path'")
RawFileURLTemplate = back
ret, err = DownloadHubIdx(&csconfig.CscliCfg{HubIndexFile: "/does/not/exist/index.json"})
ret, err = DownloadHubIdx(&csconfig.Hub{HubIndexFile: "/does/not/exist/index.json"})
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
log.Errorf("unexpected error %s", err)
}

18
pkg/cwhub/install.go
View file

@ -11,9 +11,9 @@ import (
)
//DisableItem to disable an item managed by the hub, removes the symlink if purge is true
func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool) (Item, error) {
var tdir = cscli.ConfigDir
var hdir = cscli.HubDir
func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item, error) {
var tdir = hub.ConfigDir
var hdir = hub.HubDir
syml, err := filepath.Abs(tdir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
if err != nil {
@ -34,7 +34,7 @@ func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool)
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if val, ok := hubIdx[ptrtype][p]; ok {
hubIdx[ptrtype][p], err = DisableItem(cscli, val, purge, force)
hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
}
@ -51,7 +51,7 @@ func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool)
return target, fmt.Errorf("can't delete %s : %s doesn't exist", target.Name, syml)
}
} else {
//if it's managed by hub, it's a symlink to csconfig.GConfig.Cscli.HubDir / ...
//if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", target.Name, syml)
return target, fmt.Errorf("%s isn't managed by hub", target.Name)
@ -90,9 +90,9 @@ func DisableItem(cscli *csconfig.CscliCfg, target Item, purge bool, force bool)
return target, nil
}
func EnableItem(cscli *csconfig.CscliCfg, target Item) (Item, error) {
var tdir = cscli.ConfigDir
var hdir = cscli.HubDir
func EnableItem(hub *csconfig.Hub, target Item) (Item, error) {
var tdir = hub.ConfigDir
var hdir = hub.HubDir
var err error
parent_dir := filepath.Clean(tdir + "/" + target.Type + "/" + target.Stage + "/")
/*create directories if needed*/
@ -123,7 +123,7 @@ func EnableItem(cscli *csconfig.CscliCfg, target Item) (Item, error) {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if val, ok := hubIdx[ptrtype][p]; ok {
hubIdx[ptrtype][p], err = EnableItem(cscli, val)
hubIdx[ptrtype][p], err = EnableItem(hub, val)
if err != nil {
return target, errors.Wrap(err, fmt.Sprintf("while installing %s", p))
}

32
pkg/cwhub/loader.go
View file

@ -49,7 +49,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
subs := strings.Split(path, "/")
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir)
/*we're in hub (~/.cscli/hub/)*/
/*we're in hub (~/.hub/hub/)*/
if strings.HasPrefix(path, hubdir) {
log.Tracef("in hub dir")
inhub = true
@ -95,7 +95,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
/*
we can encounter 'collections' in the form of a symlink :
/etc/crowdsec/.../collections/linux.yaml -> ~/.cscli/hub/collections/.../linux.yaml
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
when the collection is installed, both files are created
*/
//non symlinks are local user files or hub files
@ -108,7 +108,7 @@ func parser_visit(path string, f os.FileInfo, err error) error {
if err != nil {
return fmt.Errorf("unable to read symlink of %s", path)
}
//the symlink target doesn't exist, user might have remove ~/.cscli/hub/...yaml without deleting /etc/crowdsec/....yaml
//the symlink target doesn't exist, user might have remove ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
_, err := os.Lstat(hubpath)
if os.IsNotExist(err) {
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
@ -294,10 +294,10 @@ func CollecDepsCheck(v *Item) error {
return nil
}
func SyncDir(cscli *csconfig.CscliCfg, dir string) error {
hubdir = cscli.HubDir
installdir = cscli.ConfigDir
indexpath = cscli.HubIndexFile
func SyncDir(hub *csconfig.Hub, dir string) error {
hubdir = hub.HubDir
installdir = hub.ConfigDir
indexpath = hub.HubIndexFile
/*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/
for _, scan := range ItemTypes {
@ -322,13 +322,13 @@ func SyncDir(cscli *csconfig.CscliCfg, dir string) error {
}
/* Updates the infos from HubInit() with the local state */
func LocalSync(cscli *csconfig.CscliCfg) error {
func LocalSync(hub *csconfig.Hub) error {
skippedLocal = 0
skippedTainted = 0
for _, dir := range []string{cscli.ConfigDir, cscli.HubDir} {
for _, dir := range []string{hub.ConfigDir, hub.HubDir} {
log.Debugf("scanning %s", dir)
if err := SyncDir(cscli, dir); err != nil {
if err := SyncDir(hub, dir); err != nil {
return fmt.Errorf("failed to scan %s : %s", dir, err)
}
}
@ -336,12 +336,12 @@ func LocalSync(cscli *csconfig.CscliCfg) error {
return nil
}
func GetHubIdx(cscli *csconfig.CscliCfg) error {
if cscli == nil {
return fmt.Errorf("no configuration found for cscli")
func GetHubIdx(hub *csconfig.Hub) error {
if hub == nil {
return fmt.Errorf("no configuration found for hub")
}
log.Debugf("loading hub idx %s", cscli.HubIndexFile)
bidx, err := ioutil.ReadFile(cscli.HubIndexFile)
log.Debugf("loading hub idx %s", hub.HubIndexFile)
bidx, err := ioutil.ReadFile(hub.HubIndexFile)
if err != nil {
return errors.Wrap(err, "unable to read index file")
}
@ -353,7 +353,7 @@ func GetHubIdx(cscli *csconfig.CscliCfg) error {
return err
}
hubIdx = ret
if err := LocalSync(cscli); err != nil {
if err := LocalSync(hub); err != nil {
log.Fatalf("Failed to sync Hub index with local deployment : %v", err)
}
return nil

1
pkg/database/database.go
View file

@ -69,7 +69,6 @@ func NewClient(config *csconfig.DatabaseCfg) (*Client, error) {
if err := types.ConfigureLogger(clog); err != nil {
return nil, errors.Wrap(err, "while configuring db logger")
}
log.Infof("Log level: %+v", config.LogLevel)
if config.LogLevel != nil {
clog.SetLevel(*config.LogLevel)
if *config.LogLevel >= log.DebugLevel {

5
pkg/parser/unix_parser.go
View file

@ -1,8 +1,8 @@
package parser
import (
"io/ioutil"
"fmt"
"io/ioutil"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@ -45,8 +45,7 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
return &r, nil
}
func LoadParsers(cConfig *csconfig.GlobalConfig, parsers *Parsers) (*Parsers, error) {
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error
log.Infof("Loading grok library %s", cConfig.Crowdsec.ConfigDir+string("/patterns/"))

43
scripts/func_tests/config/config.yaml Normal file
View file

@ -0,0 +1,43 @@
common:
daemonize: true
pid_dir: /var/run/
log_media: file
log_level: info
log_dir: /var/log/
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/
simulation_path: /etc/crowdsec/simulation.yaml
hub_dir: /etc/crowdsec/hub/
index_path: /etc/crowdsec/hub/.index.json
crowdsec_service:
acquisition_path: /etc/crowdsec/acquis.yaml
parser_routines: 1
cscli:
output: human
db_config:
log_level: info
type: sqlite
db_path: /var/lib/crowdsec/data/crowdsec.db
flush:
max_items: 5000
max_age: 7d
api:
client:
insecure_skip_verify: false
credentials_path: /etc/crowdsec/local_api_credentials.yaml
server:
log_level: info
listen_uri: 127.0.0.1:8080
profiles_path: /etc/crowdsec/profiles.yaml
online_client: # Crowdsec API credentials (to push signals and receive bad IPs)
credentials_path: /etc/crowdsec/online_api_credentials.yaml
# tls:
# cert_file: /etc/crowdsec/ssl/cert.pem
# key_file: /etc/crowdsec/ssl/key.pem
prometheus:
enabled: true
level: full
listen_addr: 127.0.0.1
listen_port: 6060

41
scripts/func_tests/config/config_no_agent.yaml Normal file
View file

@ -0,0 +1,41 @@
common:
daemonize: true
pid_dir: /var/run/
log_media: file
log_level: info
log_dir: ./
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/
simulation_path: /etc/crowdsec/simulation.yaml
hub_dir: /etc/crowdsec/hub/
index_path: /etc/crowdsec/hub/.index.json
cscli:
output: human
db_config:
log_level: info
type: sqlite
db_path: /var/lib/crowdsec/data/crowdsec.db
flush:
max_items: 5000
max_age: 7d
api:
client:
insecure_skip_verify: false
credentials_path: /etc/crowdsec/local_api_credentials.yaml
server:
log_level: info
listen_uri: 127.0.0.1:8080
profiles_path: /etc/crowdsec/profiles.yaml
online_client: # Crowdsec API credentials (to push signals and receive bad IPs)
credentials_path: /etc/crowdsec/online_api_credentials.yaml
# tls:
# cert_file: /etc/crowdsec/ssl/cert.pem
# key_file: /etc/crowdsec/ssl/key.pem
prometheus:
enabled: true
level: full
listen_addr: 127.0.0.1
listen_port: 6060

38
scripts/func_tests/config/config_no_capi.yaml Normal file
View file

@ -0,0 +1,38 @@
common:
daemonize: true
pid_dir: /var/run/
log_media: file
log_level: info
log_dir: ./
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/
simulation_path: /etc/crowdsec/simulation.yaml
hub_dir: /etc/crowdsec/hub/
index_path: /etc/crowdsec/hub/.index.json
crowdsec_service:
acquisition_path: /etc/crowdsec/acquis.yaml
parser_routines: 1
cscli:
output: human
db_config:
log_level: info
type: sqlite
db_path: /var/lib/crowdsec/data/crowdsec.db
flush:
max_items: 5000
max_age: 7d
api:
client:
insecure_skip_verify: false
credentials_path: /etc/crowdsec/local_api_credentials.yaml
server:
log_level: info
listen_uri: 127.0.0.1:8080
profiles_path: /etc/crowdsec/profiles.yaml
prometheus:
enabled: true
level: full
listen_addr: 127.0.0.1
listen_port: 6060

34
scripts/func_tests/config/config_no_lapi.yaml Normal file
View file

@ -0,0 +1,34 @@
common:
daemonize: true
pid_dir: /var/run/
log_media: file
log_level: info
log_dir: ./
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/
simulation_path: /etc/crowdsec/simulation.yaml
hub_dir: /etc/crowdsec/hub/
index_path: /etc/crowdsec/hub/.index.json
crowdsec_service:
acquisition_path: /etc/crowdsec/acquis.yaml
parser_routines: 1
cscli:
output: human
db_config:
log_level: info
type: sqlite
db_path: /var/lib/crowdsec/data/crowdsec.db
flush:
max_items: 5000
max_age: 7d
api:
client:
insecure_skip_verify: false
credentials_path: /etc/crowdsec/local_api_credentials.yaml
prometheus:
enabled: true
level: full
listen_addr: 127.0.0.1
listen_port: 6060

15
scripts/func_tests/systemd/crowdsec.service Normal file
View file

@ -0,0 +1,15 @@
[Unit]
Description=Crowdsec agent
After=syslog.target network.target remote-fs.target nss-lookup.target
[Service]
Type=notify
Environment=LC_ALL=C LANG=C
PIDFile=/var/run/crowdsec.pid
ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml
#ExecStartPost=/bin/sleep 0.1
ExecReload=/bin/kill -HUP $MAINPID
[Install]
WantedBy=multi-user.target

15
scripts/func_tests/systemd/crowdsec_no_agent.service Normal file
View file

@ -0,0 +1,15 @@
[Unit]
Description=Crowdsec agent
After=syslog.target network.target remote-fs.target nss-lookup.target
[Service]
Type=notify
Environment=LC_ALL=C LANG=C
PIDFile=/var/run/crowdsec.pid
ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -no-cs
#ExecStartPost=/bin/sleep 0.1
ExecReload=/bin/kill -HUP $MAINPID
[Install]
WantedBy=multi-user.target

15
scripts/func_tests/systemd/crowdsec_no_lapi.service Normal file
View file

@ -0,0 +1,15 @@
[Unit]
Description=Crowdsec agent
After=syslog.target network.target remote-fs.target nss-lookup.target
[Service]
Type=notify
Environment=LC_ALL=C LANG=C
PIDFile=/var/run/crowdsec.pid
ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -no-api
#ExecStartPost=/bin/sleep 0.1
ExecReload=/bin/kill -HUP $MAINPID
[Install]
WantedBy=multi-user.target

2
scripts/func_tests/tests_base.sh
View file

@ -13,7 +13,7 @@ JQ="jq -e"
SYSTEMCTL="sudo systemctl --no-pager"
CROWDSEC="sudo crowdsec"
CROWDSEC_PROCESS="crowdsec"
# helpers
function fail {
echo "ACTION FAILED, STOP : $@"

124
scripts/func_tests/tests_post-install_0base.sh
View file

@ -5,9 +5,13 @@ source tests_base.sh
##########################
## TEST AGENT/LAPI/CAPI ##
echo "CROWDSEC (AGENT+LAPI+CAPI)"
## status / start / stop
# service should be up
pidof crowdsec || fail "crowdsec process shouldn't be running"
pidof crowdsec || fail "crowdsec process should be running"
${SYSTEMCTL} status crowdsec || fail "systemctl status crowdsec failed"
#shut it down
@ -18,43 +22,125 @@ pidof crowdsec && fail "crowdsec process shouldn't be running"
#start it again
${SYSTEMCTL} start crowdsec || fail "failed to stop service"
${SYSTEMCTL} status crowdsec || fail "crowdsec should be down"
pidof crowdsec || fail "crowdsec process shouldn't be running"
pidof crowdsec || fail "crowdsec process should be running"
#restart it
${SYSTEMCTL} restart crowdsec || fail "failed to stop service"
${SYSTEMCTL} status crowdsec || fail "crowdsec should be down"
pidof crowdsec || fail "crowdsec process shouldn't be running"
pidof crowdsec || fail "crowdsec process should be running"
## version
${CSCLI} version || fail "cannot run cscli version"
## alerts
# alerts list at startup should just return one entry : comunity pull
sleep 5
${CSCLI} alerts list -ojson | ${JQ} '. | length >= 1' || fail "expected at least one entry from cscli alerts list"
## capi
${CSCLI} capi status || fail "capi status should be ok"
## config
${CSCLI} config show || fail "failed to show config"
${CSCLI} config backup ./test || fail "failed to backup config"
sudo rm -rf ./test
## lapi
${CSCLI} lapi status || fail "lapi status failed"
## metrics
${CSCLI} metrics || fail "failed to get metrics"
${SYSTEMCTL} stop crowdsec || fail "crowdsec should be down"
#######################
## TEST WITHOUT LAPI ##
echo "CROWDSEC (AGENT)"
# test with -no-api flag
sudo cp ./systemd/crowdsec_no_lapi.service /etc/systemd/system/crowdsec.service
${SYSTEMCTL} daemon-reload
${SYSTEMCTL} start crowdsec
sleep 1
pidof crowdsec && fail "crowdsec shouldn't run without LAPI (in flag)"
${SYSTEMCTL} stop crowdsec
sudo cp ./systemd/crowdsec.service /etc/systemd/system/crowdsec.service
${SYSTEMCTL} daemon-reload
# test with no api server in configuration file
sudo cp ./config/config_no_lapi.yaml /etc/crowdsec/config.yaml
${SYSTEMCTL} start crowdsec
sleep 1
pidof crowdsec && fail "crowdsec agent should not run without lapi (in configuration file)"
##### cscli test ####
## capi
${CSCLI} -c ./config/config_no_lapi.yaml capi status && fail "capi status shouldn't be ok"
## config
${CSCLI_BIN} -c ./config/config_no_lapi.yaml config show || fail "failed to show config"
${CSCLI} -c ./config/config_no_lapi.yaml config backup ./test || fail "failed to backup config"
sudo rm -rf ./test
## lapi
${CSCLI} -c ./config/config_no_lapi.yaml lapi status && fail "lapi status should not be ok" ## if lapi status success, it means that the test fail
## metrics
${CSCLI_BIN} -c ./config/config_no_lapi.yaml metrics
${SYSTEMCTL} stop crowdsec
sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
########################
## TEST WITHOUT AGENT ##
echo "CROWDSEC (LAPI+CAPI)"
# test with -no-cs flag
sudo cp ./systemd/crowdsec_no_agent.service /etc/systemd/system/crowdsec.service
${SYSTEMCTL} daemon-reload
${SYSTEMCTL} start crowdsec
pidof crowdsec || fail "crowdsec LAPI should run without agent (in flag)"
${SYSTEMCTL} stop crowdsec
sudo cp ./systemd/crowdsec.service /etc/systemd/system/crowdsec.service
${SYSTEMCTL} daemon-reload
# test with no crowdsec agent in configuration file
sudo cp ./config/config_no_agent.yaml /etc/crowdsec/config.yaml
${SYSTEMCTL} start crowdsec
pidof crowdsec || fail "crowdsec LAPI should run without agent (in configuration file)"
## capi
${CSCLI} -c ./config/config_no_agent.yaml capi status || fail "capi status should be ok"
## config
${CSCLI_BIN} -c ./config/config_no_agent.yaml config show || fail "failed to show config"
${CSCLI} -c ./config/config_no_agent.yaml config backup ./test || fail "failed to backup config"
sudo rm -rf ./test
## lapi
${CSCLI} -c ./config/config_no_agent.yaml lapi status || fail "lapi status failed"
## metrics
${CSCLI_BIN} -c ./config/config_no_agent.yaml metrics || fail "failed to get metrics"
${SYSTEMCTL} stop crowdsec
sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
#######################
## TEST WITHOUT CAPI ##
echo "CROWDSEC (AGENT+LAPI)"
# test with no online client in configuration file
sudo cp ./config/config_no_capi.yaml /etc/crowdsec/config.yaml
${SYSTEMCTL} start crowdsec
pidof crowdsec || fail "crowdsec LAPI should run without CAPI (in configuration file)"
## capi
${CSCLI} -c ./config/config_no_capi.yaml capi status && fail "capi status should not be ok" ## if capi status success, it means that the test fail
## config
${CSCLI_BIN} -c ./config/config_no_capi.yaml config show || fail "failed to show config"
${CSCLI} -c ./config/config_no_capi.yaml config backup ./test || fail "failed to backup config"
sudo rm -rf ./test
## lapi
${CSCLI} -c ./config/config_no_capi.yaml lapi status || fail "lapi status failed"
## metrics
${CSCLI_BIN} -c ./config/config_no_capi.yaml metrics || fail "failed to get metrics"
sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
${SYSTEMCTL} restart crowdsec

8
scripts/func_tests/tests_post-install_2collections.sh
View file

@ -5,12 +5,12 @@ source tests_base.sh
## collections
${CSCLI} collections list || fail "failed to list collections"
${CSCLI_BIN} collections list || fail "failed to list collections"
BASE_COLLECTION_COUNT=2
# we expect 1 collections : linux
${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
# install an extra collection
${CSCLI} collections install crowdsecurity/mysql || fail "failed to install collection"
@ -18,7 +18,7 @@ ${CSCLI} collections install crowdsecurity/mysql || fail "failed to install coll
BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT+1))
# we should now have 2 collections :)
${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
# remove the collection
${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collection"
@ -26,5 +26,5 @@ ${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collec
BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT-1))
# we expect 1 collections : linux
${CSCLI} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"
${CSCLI_BIN} collections list -ojson | ${JQ} ". | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"