0ct0pu5/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Simulation support (#136)

Browse source 
* support simulation mode
This commit is contained in:
Thibault "bui" Koechlin 2020-07-16 15:59:09 +02:00 • committed by GitHub
parent 87a90583fe
commit 7fe6741df3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 328 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/crowdsec-cli/config.go
View file

@ -3,6 +3,8 @@ package main
import (
"fmt"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
@ -18,6 +20,8 @@ type cliConfig struct {
InstallFolder string
BackendPluginFolder string `yaml:"backend_folder"`
DataFolder string `yaml:"data_folder"`
SimulationCfgPath string `yaml:"simulation_path,omitempty"`
SimulationCfg *csconfig.SimulationConfig
}
func NewConfigCmd() *cobra.Command {

4
cmd/crowdsec-cli/main.go
View file

@ -63,6 +63,8 @@ func initConfig() {
cwhub.Cfgdir = config.configFolder
cwhub.Hubdir = config.HubFolder
config.configured = true
config.SimulationCfg = csConfig.SimulationCfg
config.SimulationCfgPath = csConfig.SimulationCfgPath
}
func main() {
@ -141,7 +143,7 @@ API interaction:
rootCmd.AddCommand(NewBackupCmd())
rootCmd.AddCommand(NewDashboardCmd())
rootCmd.AddCommand(NewInspectCmd())
rootCmd.AddCommand(NewSimulationCmds())
if err := rootCmd.Execute(); err != nil {
log.Fatalf("While executing root command : %s", err)
}

222
cmd/crowdsec-cli/simulation.go Normal file
View file

@ -0,0 +1,222 @@
package main
import (
"fmt"
"io/ioutil"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
)
func addToExclusion(name string) error {
config.SimulationCfg.Exclusions = append(config.SimulationCfg.Exclusions, name)
return nil
}
func removeFromExclusion(name string) error {
index := indexOf(name, config.SimulationCfg.Exclusions)
// Remove element from the slice
config.SimulationCfg.Exclusions[index] = config.SimulationCfg.Exclusions[len(config.SimulationCfg.Exclusions)-1]
config.SimulationCfg.Exclusions[len(config.SimulationCfg.Exclusions)-1] = ""
config.SimulationCfg.Exclusions = config.SimulationCfg.Exclusions[:len(config.SimulationCfg.Exclusions)-1]
return nil
}
func enableGlobalSimulation() error {
config.SimulationCfg.Simulation = true
config.SimulationCfg.Exclusions = []string{}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("unable to dump simulation file: %s", err.Error())
}
log.Printf("global simulation: enabled")
return nil
}
func dumpSimulationFile() error {
newConfigSim, err := yaml.Marshal(config.SimulationCfg)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = ioutil.WriteFile(config.SimulationCfgPath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' : %s", config.SimulationCfgPath, err)
}
return nil
}
func disableGlobalSimulation() error {
config.SimulationCfg.Simulation = false
config.SimulationCfg.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(config.SimulationCfg)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = ioutil.WriteFile(config.SimulationCfgPath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s' : %s", config.SimulationCfgPath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func simulationStatus() error {
if config.SimulationCfg == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return nil
}
if config.SimulationCfg.Simulation {
log.Println("global simulation: enabled")
if len(config.SimulationCfg.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range config.SimulationCfg.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(config.SimulationCfg.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range config.SimulationCfg.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
return nil
}
func NewSimulationCmds() *cobra.Command {
var cmdSimulation = &cobra.Command{
Use: "simulation enable|disable [scenario_name]",
Short: "",
Long: ``,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if !config.configured {
return fmt.Errorf("you must configure cli before using simulation")
}
return nil
},
}
cmdSimulation.Flags().SortFlags = false
cmdSimulation.PersistentFlags().SortFlags = false
var cmdSimulationEnable = &cobra.Command{
Use: "enable [scenario_name]",
Short: "Enable the simulation, globally or on specified scenarios",
Long: ``,
Example: `cscli simulation enable`,
Run: func(cmd *cobra.Command, args []string) {
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("failed to get Hub index : %v", err)
}
if len(args) > 0 {
for _, scenario := range args {
var v cwhub.Item
var ok bool
if _, ok = cwhub.HubIdx[cwhub.SCENARIOS]; ok {
if v, ok = cwhub.HubIdx[cwhub.SCENARIOS][scenario]; !ok {
log.Errorf("'%s' isn't present in hub index", scenario)
continue
}
if !v.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
}
isExcluded := inSlice(scenario, config.SimulationCfg.Exclusions)
if config.SimulationCfg.Simulation && !isExcluded {
log.Warningf("global simulation is already enabled")
continue
}
if !config.SimulationCfg.Simulation && isExcluded {
log.Warningf("simulation for '%s' already enabled", scenario)
continue
}
if config.SimulationCfg.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatalf(err.Error())
}
log.Printf("simulation enabled for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatalf(err.Error())
}
log.Printf("simulation mode for '%s' enabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation enable: %s", err.Error())
}
} else {
if err := enableGlobalSimulation(); err != nil {
log.Fatalf("unable to enable global simulation mode : %s", err.Error())
}
}
},
}
cmdSimulation.AddCommand(cmdSimulationEnable)
var cmdSimulationDisable = &cobra.Command{
Use: "disable [scenario_name]",
Short: "Disable the simulation mode. Disable only specified scenarios",
Long: ``,
Example: `cscli simulation disable`,
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
for _, scenario := range args {
isExcluded := inSlice(scenario, config.SimulationCfg.Exclusions)
if !config.SimulationCfg.Simulation && !isExcluded {
log.Warningf("%s isn't in simulation mode", scenario)
continue
}
if !config.SimulationCfg.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatalf(err.Error())
}
log.Printf("simulation mode for '%s' disabled", scenario)
continue
}
if isExcluded {
log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatalf(err.Error())
}
log.Printf("simulation mode for '%s' disabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation disable: %s", err.Error())
}
} else {
if err := disableGlobalSimulation(); err != nil {
log.Fatalf("unable to disable global simulation mode : %s", err.Error())
}
}
},
}
cmdSimulation.AddCommand(cmdSimulationDisable)
var cmdSimulationStatus = &cobra.Command{
Use: "status",
Short: "Show simulation mode status",
Long: ``,
Example: `cscli simulation status`,
Run: func(cmd *cobra.Command, args []string) {
if err := simulationStatus(); err != nil {
log.Fatalf(err.Error())
}
},
}
cmdSimulation.AddCommand(cmdSimulationStatus)
return cmdSimulation
}

19
cmd/crowdsec-cli/utils.go Normal file
View file

@ -0,0 +1,19 @@
package main
func inSlice(s string, slice []string) bool {
for _, str := range slice {
if s == str {
return true
}
}
return false
}
func indexOf(s string, slice []string) int {
for i, elem := range slice {
if s == elem {
return i
}
}
return -1
}

12
cmd/crowdsec/output.go
View file

@ -33,6 +33,10 @@ LOOP:
log.Infof("Done shutdown down output")
break LOOP
case event := <-overflow:
//if global simulation -> everything is simulation unless told otherwise
if cConfig.SimulationCfg != nil && cConfig.SimulationCfg.Simulation {
event.Overflow.Simulation = true
}
if cConfig.Profiling {
start = time.Now()
}
@ -47,6 +51,14 @@ LOOP:
if err != nil {
return fmt.Errorf("postoverflow failed : %s", err)
}
//check scenarios in simulation
if cConfig.SimulationCfg != nil {
for _, scenario_name := range cConfig.SimulationCfg.Exclusions {
if event.Overflow.Scenario == scenario_name {
event.Overflow.Simulation = !event.Overflow.Simulation
}
}
}
if event.Overflow.Scenario == "" && event.Overflow.MapKey != "" {
//log.Infof("Deleting expired entry %s", event.Overflow.MapKey)

5
cmd/crowdsec/serve.go
View file

@ -45,6 +45,11 @@ func reloadHandler(sig os.Signal) error {
if err := leaky.ShutdownAllBuckets(buckets); err != nil {
log.Fatalf("while shutting down routines : %s", err)
}
//reload the simulation state
if err := cConfig.LoadSimulation(); err != nil {
log.Errorf("reload error (simulation) : %s", err)
}
//reload all and start processing again :)
if err := LoadParsers(cConfig); err != nil {
log.Fatalf("Failed to load parsers: %s", err)

1
config/prod.yaml
View file

@ -4,6 +4,7 @@ config_dir: ${CFG}
pid_dir: ${PID}
log_dir: /var/log/
cscli_dir: ${CFG}/cscli
simulation_path: ${CFG}/simulation.yaml
log_mode: file
log_level: info
profiling: false

4
config/simulation.yaml Normal file
View file

@ -0,0 +1,4 @@
simulation: off
# exclusions:
# - crowdsecurity/ssh-bf

1
go.mod
View file

@ -29,6 +29,7 @@ require (
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/oschwald/geoip2-golang v1.4.0
github.com/oschwald/maxminddb-golang v1.6.0
github.com/pkg/errors v0.8.1
github.com/prometheus/client_golang v1.5.1
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.9.1

82
pkg/csconfig/config.go
View file

@ -15,29 +15,36 @@ import (
"gopkg.in/yaml.v2"
)
type SimulationConfig struct {
Simulation bool `yaml:"simulation"`
Exclusions []string `yaml:"exclusions,omitempty"`
}
// CrowdSec is the structure of the crowdsec configuration
type CrowdSec struct {
WorkingFolder string `yaml:"working_dir,omitempty"`
DataFolder string `yaml:"data_dir,omitempty"`
ConfigFolder string `yaml:"config_dir,omitempty"`
AcquisitionFile string `yaml:"acquis_path,omitempty"`
SingleFile string //for forensic mode
SingleFileLabel string //for forensic mode
PIDFolder string `yaml:"pid_dir,omitempty"`
LogFolder string `yaml:"log_dir,omitempty"`
LogMode string `yaml:"log_mode,omitempty"` //like file, syslog or stdout ?
LogLevel log.Level `yaml:"log_level,omitempty"` //trace,debug,info,warning,error
Daemonize bool `yaml:"daemon,omitempty"` //true -> go background
Profiling bool `yaml:"profiling,omitempty"` //true -> enable runtime profiling
APIMode bool `yaml:"apimode,omitempty"` //true -> enable api push
CsCliFolder string `yaml:"cscli_dir"` //cscli folder
NbParsers int `yaml:"parser_routines"` //the number of go routines to start for parsing
Linter bool
Prometheus bool
HTTPListen string `yaml:"http_listen,omitempty"`
RestoreMode string
DumpBuckets bool
OutputConfig *outputs.OutputFactory `yaml:"plugin"`
WorkingFolder string `yaml:"working_dir,omitempty"`
DataFolder string `yaml:"data_dir,omitempty"`
ConfigFolder string `yaml:"config_dir,omitempty"`
AcquisitionFile string `yaml:"acquis_path,omitempty"`
SingleFile string //for forensic mode
SingleFileLabel string //for forensic mode
PIDFolder string `yaml:"pid_dir,omitempty"`
LogFolder string `yaml:"log_dir,omitempty"`
LogMode string `yaml:"log_mode,omitempty"` //like file, syslog or stdout ?
LogLevel log.Level `yaml:"log_level,omitempty"` //trace,debug,info,warning,error
Daemonize bool `yaml:"daemon,omitempty"` //true -> go background
Profiling bool `yaml:"profiling,omitempty"` //true -> enable runtime profiling
APIMode bool `yaml:"apimode,omitempty"` //true -> enable api push
CsCliFolder string `yaml:"cscli_dir"` //cscli folder
NbParsers int `yaml:"parser_routines"` //the number of go routines to start for parsing
SimulationCfgPath string `yaml:"simulation_path,omitempty"`
SimulationCfg *SimulationConfig
Linter bool
Prometheus bool
HTTPListen string `yaml:"http_listen,omitempty"`
RestoreMode string
DumpBuckets bool
OutputConfig *outputs.OutputFactory `yaml:"plugin"`
}
// NewCrowdSecConfig create a new crowdsec configuration with default configuration
@ -59,6 +66,21 @@ func NewCrowdSecConfig() *CrowdSec {
}
}
func (c *CrowdSec) LoadSimulation() error {
if c.SimulationCfgPath != "" {
rcfg, err := ioutil.ReadFile(c.SimulationCfgPath)
if err != nil {
return fmt.Errorf("while reading '%s' : %s", c.SimulationCfgPath, err)
}
simCfg := SimulationConfig{}
if err := yaml.UnmarshalStrict(rcfg, &simCfg); err != nil {
return fmt.Errorf("while parsing '%s' : %s", c.SimulationCfgPath, err)
}
c.SimulationCfg = &simCfg
}
return nil
}
func (c *CrowdSec) GetCliConfig(configFile *string) error {
/*overriden by cfg file*/
if *configFile != "" {
@ -73,8 +95,10 @@ func (c *CrowdSec) GetCliConfig(configFile *string) error {
c.AcquisitionFile = filepath.Clean(c.ConfigFolder + "/acquis.yaml")
}
}
if err := c.LoadSimulation(); err != nil {
return fmt.Errorf("loading simulation config : %s", err)
}
return nil
}
// GetOPT return flags parsed from command line
@ -111,18 +135,8 @@ func (c *CrowdSec) GetOPT() error {
c.SingleFileLabel = *catFileType
}
/*overriden by cfg file*/
if *configFile != "" {
rcfg, err := ioutil.ReadFile(*configFile)
if err != nil {
return fmt.Errorf("read '%s' : %s", *configFile, err)
}
if err := yaml.UnmarshalStrict(rcfg, c); err != nil {
return fmt.Errorf("parse '%s' : %s", *configFile, err)
}
if c.AcquisitionFile == "" {
c.AcquisitionFile = filepath.Clean(c.ConfigFolder + "/acquis.yaml")
}
if err := c.GetCliConfig(configFile); err != nil {
log.Fatalf("Error while loading configuration : %s", err)
}
if *AcquisitionFile != "" {

10
pkg/outputs/ouputs.go
View file

@ -45,13 +45,17 @@ func OvflwToOrder(sig types.SignalOccurence, prof types.Profile) (*types.BanOrde
var ordr types.BanOrder
var warn error
if sig.Simulation {
log.Debugf("signal for '%s' is whitelisted", sig.Source_ip)
ordr.MeasureType = "simulation:"
}
//Identify remediation type
if prof.Remediation.Ban {
ordr.MeasureType = "ban"
ordr.MeasureType += "ban"
} else if prof.Remediation.Slow {
ordr.MeasureType = "slow"
ordr.MeasureType += "slow"
} else if prof.Remediation.Captcha {
ordr.MeasureType = "captcha"
ordr.MeasureType += "captcha"
} else {
/*if the profil has no remediation, no order */
return nil, nil, fmt.Errorf("no remediation")

1
pkg/types/signal_occurence.go
View file

@ -38,6 +38,7 @@ type SignalOccurence struct {
Capacity int `json:"capacity,omitempty"`
Leak_speed time.Duration `json:"leak_speed,omitempty"`
Whitelisted bool `gorm:"-"`
Simulation bool `gorm:"-"`
Reprocess bool //Reprocess, when true, will make the overflow being processed again as a fresh log would
Labels map[string]string `gorm:"-"`
}

1
scripts/test_env.sh
View file

@ -73,6 +73,7 @@ create_arbo() {
copy_files() {
cp "./config/profiles.yaml" "$CONFIG_DIR"
cp "./config/dev.yaml" "$BASE"
cp "./config/simulation.yaml" "$CONFIG_DIR"
cp "./cmd/crowdsec/crowdsec" "$BASE"
cp "./cmd/crowdsec-cli/cscli" "$BASE"
cp -r "./config/patterns" "$CONFIG_DIR"