|
@@ -34,49 +34,50 @@ import (
|
|
|
// BucketFactory struct holds all fields for any bucket configuration. This is to have a
|
|
|
// generic struct for buckets. This can be seen as a bucket factory.
|
|
|
type BucketFactory struct {
|
|
|
- FormatVersion string `yaml:"format"`
|
|
|
- Author string `yaml:"author"`
|
|
|
- Description string `yaml:"description"`
|
|
|
- References []string `yaml:"references"`
|
|
|
- Type string `yaml:"type"` //Type can be : leaky, counter, trigger. It determines the main bucket characteristics
|
|
|
- Name string `yaml:"name"` //Name of the bucket, used later in log and user-messages. Should be unique
|
|
|
- Capacity int `yaml:"capacity"` //Capacity is applicable to leaky buckets and determines the "burst" capacity
|
|
|
- LeakSpeed string `yaml:"leakspeed"` //Leakspeed is a float representing how many events per second leak out of the bucket
|
|
|
- Duration string `yaml:"duration"` //Duration allows 'counter' buckets to have a fixed life-time
|
|
|
- Filter string `yaml:"filter"` //Filter is an expr that determines if an event is elligible for said bucket. Filter is evaluated against the Event struct
|
|
|
- GroupBy string `yaml:"groupby,omitempty"` //groupy is an expr that allows to determine the partitions of the bucket. A common example is the source_ip
|
|
|
- Distinct string `yaml:"distinct"` //Distinct, when present, adds a `Pour()` processor that will only pour uniq items (based on distinct expr result)
|
|
|
- Debug bool `yaml:"debug"` //Debug, when set to true, will enable debugging for _this_ scenario specifically
|
|
|
- Labels map[string]string `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow
|
|
|
- Blackhole string `yaml:"blackhole,omitempty"` //Blackhole is a duration that, if present, will prevent same bucket partition to overflow more often than $duration
|
|
|
- logger *log.Entry `yaml:"-"` //logger is bucket-specific logger (used by Debug as well)
|
|
|
- Reprocess bool `yaml:"reprocess"` //Reprocess, if true, will for the bucket to be re-injected into processing chain
|
|
|
- CacheSize int `yaml:"cache_size"` //CacheSize, if > 0, limits the size of in-memory cache of the bucket
|
|
|
- Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc.
|
|
|
- OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through
|
|
|
- ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP
|
|
|
- BucketName string `yaml:"-"`
|
|
|
- Filename string `yaml:"-"`
|
|
|
- RunTimeFilter *vm.Program `json:"-"`
|
|
|
- ExprDebugger *exprhelpers.ExprDebugger `yaml:"-" json:"-"` // used to debug expression by printing the content of each variable of the expression
|
|
|
- RunTimeGroupBy *vm.Program `json:"-"`
|
|
|
- Data []*types.DataSource `yaml:"data,omitempty"`
|
|
|
- DataDir string `yaml:"-"`
|
|
|
- CancelOnFilter string `yaml:"cancel_on,omitempty"` //a filter that, if matched, kills the bucket
|
|
|
- leakspeed time.Duration //internal representation of `Leakspeed`
|
|
|
- duration time.Duration //internal representation of `Duration`
|
|
|
- ret chan types.Event //the bucket-specific output chan for overflows
|
|
|
- processors []Processor //processors is the list of hooks for pour/overflow/create (cf. uniq, blackhole etc.)
|
|
|
- output bool //??
|
|
|
- ScenarioVersion string `yaml:"version,omitempty"`
|
|
|
- hash string `yaml:"-"`
|
|
|
- Simulated bool `yaml:"simulated"` //Set to true if the scenario instantiating the bucket was in the exclusion list
|
|
|
- tomb *tomb.Tomb `yaml:"-"`
|
|
|
- wgPour *sync.WaitGroup `yaml:"-"`
|
|
|
- wgDumpState *sync.WaitGroup `yaml:"-"`
|
|
|
+ FormatVersion string `yaml:"format"`
|
|
|
+ Author string `yaml:"author"`
|
|
|
+ Description string `yaml:"description"`
|
|
|
+ References []string `yaml:"references"`
|
|
|
+ Type string `yaml:"type"` //Type can be : leaky, counter, trigger. It determines the main bucket characteristics
|
|
|
+ Name string `yaml:"name"` //Name of the bucket, used later in log and user-messages. Should be unique
|
|
|
+ Capacity int `yaml:"capacity"` //Capacity is applicable to leaky buckets and determines the "burst" capacity
|
|
|
+ LeakSpeed string `yaml:"leakspeed"` //Leakspeed is a float representing how many events per second leak out of the bucket
|
|
|
+ Duration string `yaml:"duration"` //Duration allows 'counter' buckets to have a fixed life-time
|
|
|
+ Filter string `yaml:"filter"` //Filter is an expr that determines if an event is elligible for said bucket. Filter is evaluated against the Event struct
|
|
|
+ GroupBy string `yaml:"groupby,omitempty"` //groupy is an expr that allows to determine the partitions of the bucket. A common example is the source_ip
|
|
|
+ Distinct string `yaml:"distinct"` //Distinct, when present, adds a `Pour()` processor that will only pour uniq items (based on distinct expr result)
|
|
|
+ Debug bool `yaml:"debug"` //Debug, when set to true, will enable debugging for _this_ scenario specifically
|
|
|
+ Labels map[string]string `yaml:"labels"` //Labels is K:V list aiming at providing context the overflow
|
|
|
+ Blackhole string `yaml:"blackhole,omitempty"` //Blackhole is a duration that, if present, will prevent same bucket partition to overflow more often than $duration
|
|
|
+ logger *log.Entry `yaml:"-"` //logger is bucket-specific logger (used by Debug as well)
|
|
|
+ Reprocess bool `yaml:"reprocess"` //Reprocess, if true, will for the bucket to be re-injected into processing chain
|
|
|
+ CacheSize int `yaml:"cache_size"` //CacheSize, if > 0, limits the size of in-memory cache of the bucket
|
|
|
+ Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc.
|
|
|
+ OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through
|
|
|
+ ConditionalOverflow string `yaml:"condition"` //condition if present, is an expression that must return true for the bucket to overflow
|
|
|
+ ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP
|
|
|
+ BucketName string `yaml:"-"`
|
|
|
+ Filename string `yaml:"-"`
|
|
|
+ RunTimeFilter *vm.Program `json:"-"`
|
|
|
+ ExprDebugger *exprhelpers.ExprDebugger `yaml:"-" json:"-"` // used to debug expression by printing the content of each variable of the expression
|
|
|
+ RunTimeGroupBy *vm.Program `json:"-"`
|
|
|
+ Data []*types.DataSource `yaml:"data,omitempty"`
|
|
|
+ DataDir string `yaml:"-"`
|
|
|
+ CancelOnFilter string `yaml:"cancel_on,omitempty"` //a filter that, if matched, kills the bucket
|
|
|
+ leakspeed time.Duration //internal representation of `Leakspeed`
|
|
|
+ duration time.Duration //internal representation of `Duration`
|
|
|
+ ret chan types.Event //the bucket-specific output chan for overflows
|
|
|
+ processors []Processor //processors is the list of hooks for pour/overflow/create (cf. uniq, blackhole etc.)
|
|
|
+ output bool //??
|
|
|
+ ScenarioVersion string `yaml:"version,omitempty"`
|
|
|
+ hash string `yaml:"-"`
|
|
|
+ Simulated bool `yaml:"simulated"` //Set to true if the scenario instantiating the bucket was in the exclusion list
|
|
|
+ tomb *tomb.Tomb `yaml:"-"`
|
|
|
+ wgPour *sync.WaitGroup `yaml:"-"`
|
|
|
+ wgDumpState *sync.WaitGroup `yaml:"-"`
|
|
|
}
|
|
|
|
|
|
-//we use one NameGenerator for all the future buckets
|
|
|
+// we use one NameGenerator for all the future buckets
|
|
|
var seed namegenerator.Generator = namegenerator.NewNameGenerator(time.Now().UTC().UnixNano())
|
|
|
|
|
|
func ValidateFactory(bucketFactory *BucketFactory) error {
|
|
@@ -98,7 +99,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
|
|
}
|
|
|
} else if bucketFactory.Type == "counter" {
|
|
|
if bucketFactory.Duration == "" {
|
|
|
- return fmt.Errorf("duration ca't be empty for counter")
|
|
|
+ return fmt.Errorf("duration can't be empty for counter")
|
|
|
}
|
|
|
if bucketFactory.duration == 0 {
|
|
|
return fmt.Errorf("bad duration for counter bucket '%d'", bucketFactory.duration)
|
|
@@ -110,6 +111,19 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
|
|
|
if bucketFactory.Capacity != 0 {
|
|
|
return fmt.Errorf("trigger bucket must have 0 capacity")
|
|
|
}
|
|
|
+ } else if bucketFactory.Type == "conditional" {
|
|
|
+ if bucketFactory.ConditionalOverflow == "" {
|
|
|
+ return fmt.Errorf("conditional bucket must have a condition")
|
|
|
+ }
|
|
|
+ if bucketFactory.Capacity != -1 {
|
|
|
+ bucketFactory.logger.Warnf("Using a value different than -1 as capacity for conditional bucket, this may lead to unexpected overflows")
|
|
|
+ }
|
|
|
+ if bucketFactory.LeakSpeed == "" {
|
|
|
+ return fmt.Errorf("leakspeed can't be empty for conditional bucket")
|
|
|
+ }
|
|
|
+ if bucketFactory.leakspeed == 0 {
|
|
|
+ return fmt.Errorf("bad leakspeed for conditional bucket '%s'", bucketFactory.LeakSpeed)
|
|
|
+ }
|
|
|
} else {
|
|
|
return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type)
|
|
|
}
|
|
@@ -304,6 +318,8 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
|
|
|
bucketFactory.processors = append(bucketFactory.processors, &Trigger{})
|
|
|
case "counter":
|
|
|
bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{})
|
|
|
+ case "conditional":
|
|
|
+ bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{})
|
|
|
default:
|
|
|
return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err)
|
|
|
}
|
|
@@ -338,6 +354,16 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
|
|
|
bucketFactory.processors = append(bucketFactory.processors, blackhole)
|
|
|
}
|
|
|
|
|
|
+ if bucketFactory.ConditionalOverflow != "" {
|
|
|
+ bucketFactory.logger.Tracef("Adding conditional overflow.")
|
|
|
+ condovflw, err := NewConditionalOverflow(bucketFactory)
|
|
|
+ if err != nil {
|
|
|
+ bucketFactory.logger.Errorf("Error creating conditional overflow : %s", err)
|
|
|
+ return fmt.Errorf("error creating conditional overflow : %s", err)
|
|
|
+ }
|
|
|
+ bucketFactory.processors = append(bucketFactory.processors, condovflw)
|
|
|
+ }
|
|
|
+
|
|
|
if len(bucketFactory.Data) > 0 {
|
|
|
for _, data := range bucketFactory.Data {
|
|
|
if data.DestPath == "" {
|