node.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656
  1. package parser
  2. import (
  3. "fmt"
  4. "net"
  5. "strings"
  6. "time"
  7. "github.com/antonmedv/expr"
  8. "github.com/crowdsecurity/grokky"
  9. "github.com/pkg/errors"
  10. yaml "gopkg.in/yaml.v2"
  11. "github.com/antonmedv/expr/vm"
  12. "github.com/crowdsecurity/crowdsec/pkg/cache"
  13. "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
  14. "github.com/crowdsecurity/crowdsec/pkg/types"
  15. "github.com/davecgh/go-spew/spew"
  16. "github.com/prometheus/client_golang/prometheus"
  17. "github.com/sirupsen/logrus"
  18. log "github.com/sirupsen/logrus"
  19. )
  20. type Node struct {
  21. FormatVersion string `yaml:"format"`
  22. //Enable config + runtime debug of node via config o/
  23. Debug bool `yaml:"debug,omitempty"`
  24. //If enabled, the node (and its child) will report their own statistics
  25. Profiling bool `yaml:"profiling,omitempty"`
  26. //Name, author, description and reference(s) for parser pattern
  27. Name string `yaml:"name,omitempty"`
  28. Author string `yaml:"author,omitempty"`
  29. Description string `yaml:"description,omitempty"`
  30. References []string `yaml:"references,omitempty"`
  31. //if debug is present in the node, keep its specific Logger in runtime structure
  32. Logger *log.Entry `yaml:"-"`
  33. //This is mostly a hack to make writing less repetitive.
  34. //relying on stage, we know which field to parse, and we
  35. //can also promote log to next stage on success
  36. Stage string `yaml:"stage,omitempty"`
  37. //OnSuccess allows to tag a node to be able to move log to next stage on success
  38. OnSuccess string `yaml:"onsuccess,omitempty"`
  39. rn string //this is only for us in debug, a random generated name for each node
  40. //Filter is executed at runtime (with current log line as context)
  41. //and must succeed or node is exited
  42. Filter string `yaml:"filter,omitempty"`
  43. RunTimeFilter *vm.Program `yaml:"-" json:"-"` //the actual compiled filter
  44. ExprDebugger *exprhelpers.ExprDebugger `yaml:"-" json:"-"` //used to debug expression by printing the content of each variable of the expression
  45. //If node has leafs, execute all of them until one asks for a 'break'
  46. LeavesNodes []Node `yaml:"nodes,omitempty"`
  47. //Flag used to describe when to 'break' or return an 'error'
  48. EnrichFunctions EnricherCtx
  49. /* If the node is actually a leaf, it can have : grok, enrich, statics */
  50. //pattern_syntax are named grok patterns that are re-utilized over several grok patterns
  51. SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
  52. //Holds a grok pattern
  53. Grok types.GrokPattern `yaml:"grok,omitempty"`
  54. //Statics can be present in any type of node and is executed last
  55. Statics []types.ExtraField `yaml:"statics,omitempty"`
  56. //Stash allows to capture data from the log line and store it in an accessible cache
  57. Stash []types.DataCapture `yaml:"stash,omitempty"`
  58. //Whitelists
  59. Whitelist Whitelist `yaml:"whitelist,omitempty"`
  60. Data []*types.DataSource `yaml:"data,omitempty"`
  61. }
  62. func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
  63. //stage is being set automagically
  64. if n.Stage == "" {
  65. return fmt.Errorf("stage needs to be an existing stage")
  66. }
  67. /* "" behaves like continue */
  68. if n.OnSuccess != "continue" && n.OnSuccess != "next_stage" && n.OnSuccess != "" {
  69. return fmt.Errorf("onsuccess '%s' not continue,next_stage", n.OnSuccess)
  70. }
  71. if n.Filter != "" && n.RunTimeFilter == nil {
  72. return fmt.Errorf("non-empty filter '%s' was not compiled", n.Filter)
  73. }
  74. if n.Grok.RunTimeRegexp != nil || n.Grok.TargetField != "" {
  75. if n.Grok.TargetField == "" && n.Grok.ExpValue == "" {
  76. return fmt.Errorf("grok requires 'expression' or 'apply_on'")
  77. }
  78. if n.Grok.RegexpName == "" && n.Grok.RegexpValue == "" {
  79. return fmt.Errorf("grok needs 'pattern' or 'name'")
  80. }
  81. }
  82. for idx, static := range n.Statics {
  83. if static.Method != "" {
  84. if static.ExpValue == "" {
  85. return fmt.Errorf("static %d : when method is set, expression must be present", idx)
  86. }
  87. if _, ok := ectx.Registered[static.Method]; !ok {
  88. log.Warningf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
  89. }
  90. } else {
  91. if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {
  92. return fmt.Errorf("static %d : at least one of meta/event/target must be set", idx)
  93. }
  94. if static.Value == "" && static.RunTimeValue == nil {
  95. return fmt.Errorf("static %d value or expression must be set", idx)
  96. }
  97. }
  98. }
  99. for idx, stash := range n.Stash {
  100. if stash.Name == "" {
  101. return fmt.Errorf("stash %d : name must be set", idx)
  102. }
  103. if stash.Value == "" {
  104. return fmt.Errorf("stash %s : value expression must be set", stash.Name)
  105. }
  106. if stash.Key == "" {
  107. return fmt.Errorf("stash %s : key expression must be set", stash.Name)
  108. }
  109. if stash.TTL == "" {
  110. return fmt.Errorf("stash %s : ttl must be set", stash.Name)
  111. }
  112. //should be configurable
  113. if stash.MaxMapSize == 0 {
  114. stash.MaxMapSize = 100
  115. }
  116. }
  117. return nil
  118. }
  119. func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[string]interface{}) (bool, error) {
  120. var NodeState bool
  121. var NodeHasOKGrok bool
  122. clog := n.Logger
  123. cachedExprEnv := expressionEnv
  124. clog.Tracef("Event entering node")
  125. if n.RunTimeFilter != nil {
  126. //Evaluate node's filter
  127. output, err := expr.Run(n.RunTimeFilter, cachedExprEnv)
  128. if err != nil {
  129. clog.Warningf("failed to run filter : %v", err)
  130. clog.Debugf("Event leaving node : ko")
  131. return false, nil
  132. }
  133. switch out := output.(type) {
  134. case bool:
  135. if n.Debug {
  136. n.ExprDebugger.Run(clog, out, cachedExprEnv)
  137. }
  138. if !out {
  139. clog.Debugf("Event leaving node : ko (failed filter)")
  140. return false, nil
  141. }
  142. default:
  143. clog.Warningf("Expr '%s' returned non-bool, abort : %T", n.Filter, output)
  144. clog.Debugf("Event leaving node : ko")
  145. return false, nil
  146. }
  147. NodeState = true
  148. } else {
  149. clog.Tracef("Node has not filter, enter")
  150. NodeState = true
  151. }
  152. if n.Name != "" {
  153. NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
  154. }
  155. isWhitelisted := false
  156. hasWhitelist := false
  157. var srcs []net.IP
  158. /*overflow and log don't hold the source ip in the same field, should be changed */
  159. /* perform whitelist checks for ips, cidr accordingly */
  160. /* TODO move whitelist elsewhere */
  161. if p.Type == types.LOG {
  162. if _, ok := p.Meta["source_ip"]; ok {
  163. srcs = append(srcs, net.ParseIP(p.Meta["source_ip"]))
  164. }
  165. } else if p.Type == types.OVFLW {
  166. for k := range p.Overflow.Sources {
  167. srcs = append(srcs, net.ParseIP(k))
  168. }
  169. }
  170. for _, src := range srcs {
  171. if isWhitelisted {
  172. break
  173. }
  174. for _, v := range n.Whitelist.B_Ips {
  175. if v.Equal(src) {
  176. clog.Debugf("Event from [%s] is whitelisted by IP (%s), reason [%s]", src, v, n.Whitelist.Reason)
  177. isWhitelisted = true
  178. } else {
  179. clog.Tracef("whitelist: %s is not eq [%s]", src, v)
  180. }
  181. hasWhitelist = true
  182. }
  183. for _, v := range n.Whitelist.B_Cidrs {
  184. if v.Contains(src) {
  185. clog.Debugf("Event from [%s] is whitelisted by CIDR (%s), reason [%s]", src, v, n.Whitelist.Reason)
  186. isWhitelisted = true
  187. } else {
  188. clog.Tracef("whitelist: %s not in [%s]", src, v)
  189. }
  190. hasWhitelist = true
  191. }
  192. }
  193. if isWhitelisted {
  194. p.Whitelisted = true
  195. }
  196. /* run whitelist expression tests anyway */
  197. for eidx, e := range n.Whitelist.B_Exprs {
  198. output, err := expr.Run(e.Filter, cachedExprEnv)
  199. if err != nil {
  200. clog.Warningf("failed to run whitelist expr : %v", err)
  201. clog.Debug("Event leaving node : ko")
  202. return false, nil
  203. }
  204. switch out := output.(type) {
  205. case bool:
  206. if n.Debug {
  207. e.ExprDebugger.Run(clog, out, cachedExprEnv)
  208. }
  209. if out {
  210. clog.Debugf("Event is whitelisted by expr, reason [%s]", n.Whitelist.Reason)
  211. p.Whitelisted = true
  212. isWhitelisted = true
  213. }
  214. hasWhitelist = true
  215. default:
  216. log.Errorf("unexpected type %t (%v) while running '%s'", output, output, n.Whitelist.Exprs[eidx])
  217. }
  218. }
  219. if isWhitelisted {
  220. p.WhitelistReason = n.Whitelist.Reason
  221. /*huglily wipe the ban order if the event is whitelisted and it's an overflow */
  222. if p.Type == types.OVFLW { /*don't do this at home kids */
  223. ips := []string{}
  224. for _, src := range srcs {
  225. ips = append(ips, src.String())
  226. }
  227. clog.Infof("Ban for %s whitelisted, reason [%s]", strings.Join(ips, ","), n.Whitelist.Reason)
  228. p.Overflow.Whitelisted = true
  229. }
  230. }
  231. //Process grok if present, should be exclusive with nodes :)
  232. gstr := ""
  233. if n.Grok.RunTimeRegexp != nil {
  234. clog.Tracef("Processing grok pattern : %s : %p", n.Grok.RegexpName, n.Grok.RunTimeRegexp)
  235. //for unparsed, parsed etc. set sensible defaults to reduce user hassle
  236. if n.Grok.TargetField != "" {
  237. //it's a hack to avoid using real reflect
  238. if n.Grok.TargetField == "Line.Raw" {
  239. gstr = p.Line.Raw
  240. } else if val, ok := p.Parsed[n.Grok.TargetField]; ok {
  241. gstr = val
  242. } else {
  243. clog.Debugf("(%s) target field '%s' doesn't exist in %v", n.rn, n.Grok.TargetField, p.Parsed)
  244. NodeState = false
  245. }
  246. } else if n.Grok.RunTimeValue != nil {
  247. output, err := expr.Run(n.Grok.RunTimeValue, cachedExprEnv)
  248. if err != nil {
  249. clog.Warningf("failed to run RunTimeValue : %v", err)
  250. NodeState = false
  251. }
  252. switch out := output.(type) {
  253. case string:
  254. gstr = out
  255. default:
  256. clog.Errorf("unexpected return type for RunTimeValue : %T", output)
  257. }
  258. }
  259. var groklabel string
  260. if n.Grok.RegexpName == "" {
  261. groklabel = fmt.Sprintf("%5.5s...", n.Grok.RegexpValue)
  262. } else {
  263. groklabel = n.Grok.RegexpName
  264. }
  265. grok := n.Grok.RunTimeRegexp.Parse(gstr)
  266. if len(grok) > 0 {
  267. /*tag explicitly that the *current* node had a successful grok pattern. it's important to know success state*/
  268. NodeHasOKGrok = true
  269. clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok))
  270. //We managed to grok stuff, merged into parse
  271. for k, v := range grok {
  272. clog.Debugf("\t.Parsed['%s'] = '%s'", k, v)
  273. p.Parsed[k] = v
  274. }
  275. // if the grok succeed, process associated statics
  276. err := n.ProcessStatics(n.Grok.Statics, p)
  277. if err != nil {
  278. clog.Errorf("(%s) Failed to process statics : %v", n.rn, err)
  279. return false, err
  280. }
  281. } else {
  282. //grok failed, node failed
  283. clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr)
  284. NodeState = false
  285. }
  286. } else {
  287. clog.Tracef("! No grok pattern : %p", n.Grok.RunTimeRegexp)
  288. }
  289. //Process the stash (data collection) if : a grok was present and succeeded, or if there is no grok
  290. if NodeHasOKGrok || n.Grok.RunTimeRegexp == nil {
  291. for idx, stash := range n.Stash {
  292. var value string
  293. var key string
  294. if stash.ValueExpression == nil {
  295. clog.Warningf("Stash %d has no value expression, skipping", idx)
  296. continue
  297. }
  298. if stash.KeyExpression == nil {
  299. clog.Warningf("Stash %d has no key expression, skipping", idx)
  300. continue
  301. }
  302. //collect the data
  303. output, err := expr.Run(stash.ValueExpression, cachedExprEnv)
  304. if err != nil {
  305. clog.Warningf("Error while running stash val expression : %v", err)
  306. }
  307. //can we expect anything else than a string ?
  308. switch output := output.(type) {
  309. case string:
  310. value = output
  311. default:
  312. clog.Warningf("unexpected type %t (%v) while running '%s'", output, output, stash.Value)
  313. continue
  314. }
  315. //collect the key
  316. output, err = expr.Run(stash.KeyExpression, cachedExprEnv)
  317. if err != nil {
  318. clog.Warningf("Error while running stash key expression : %v", err)
  319. }
  320. //can we expect anything else than a string ?
  321. switch output := output.(type) {
  322. case string:
  323. key = output
  324. default:
  325. clog.Warningf("unexpected type %t (%v) while running '%s'", output, output, stash.Key)
  326. continue
  327. }
  328. cache.SetKey(stash.Name, key, value, &stash.TTLVal)
  329. }
  330. }
  331. //Iterate on leafs
  332. if len(n.LeavesNodes) > 0 {
  333. for _, leaf := range n.LeavesNodes {
  334. ret, err := leaf.process(p, ctx, cachedExprEnv)
  335. if err != nil {
  336. clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err)
  337. clog.Debugf("Event leaving node : ko")
  338. return false, err
  339. }
  340. clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess)
  341. if ret {
  342. NodeState = true
  343. /* if child is successful, stop processing */
  344. if n.OnSuccess == "next_stage" {
  345. clog.Debugf("child is success, OnSuccess=next_stage, skip")
  346. break
  347. }
  348. } else if !NodeHasOKGrok {
  349. /*
  350. If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails.
  351. If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success.
  352. */
  353. NodeState = false
  354. }
  355. }
  356. }
  357. /*todo : check if a node made the state change ?*/
  358. /* should the childs inherit the on_success behavior */
  359. clog.Tracef("State after nodes : %v", NodeState)
  360. //grok or leafs failed, don't process statics
  361. if !NodeState {
  362. if n.Name != "" {
  363. NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
  364. }
  365. clog.Debugf("Event leaving node : ko")
  366. return NodeState, nil
  367. }
  368. if n.Name != "" {
  369. NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
  370. }
  371. /*
  372. Please kill me. this is to apply statics when the node *has* whitelists that successfully matched the node.
  373. */
  374. if hasWhitelist && isWhitelisted && len(n.Statics) > 0 || len(n.Statics) > 0 && !hasWhitelist {
  375. clog.Debugf("+ Processing %d statics", len(n.Statics))
  376. // if all else is good in whitelist, process node's statics
  377. err := n.ProcessStatics(n.Statics, p)
  378. if err != nil {
  379. clog.Errorf("Failed to process statics : %v", err)
  380. return false, err
  381. }
  382. } else {
  383. clog.Tracef("! No node statics")
  384. }
  385. if NodeState {
  386. clog.Debugf("Event leaving node : ok")
  387. log.Tracef("node is successful, check strategy")
  388. if n.OnSuccess == "next_stage" {
  389. idx := stageidx(p.Stage, ctx.Stages)
  390. //we're at the last stage
  391. if idx+1 == len(ctx.Stages) {
  392. clog.Debugf("node reached the last stage : %s", p.Stage)
  393. } else {
  394. clog.Debugf("move Event from stage %s to %s", p.Stage, ctx.Stages[idx+1])
  395. p.Stage = ctx.Stages[idx+1]
  396. }
  397. } else {
  398. clog.Tracef("no strategy on success (%s), continue !", n.OnSuccess)
  399. }
  400. } else {
  401. clog.Debugf("Event leaving node : ko")
  402. }
  403. clog.Tracef("Node successful, continue")
  404. return NodeState, nil
  405. }
  406. func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
  407. var err error
  408. var valid bool
  409. valid = false
  410. dumpr := spew.ConfigState{MaxDepth: 1, DisablePointerAddresses: true}
  411. n.rn = seed.Generate()
  412. n.EnrichFunctions = ectx
  413. log.Tracef("compile, node is %s", n.Stage)
  414. /* if the node has debugging enabled, create a specific logger with debug
  415. that will be used only for processing this node ;) */
  416. if n.Debug {
  417. var clog = logrus.New()
  418. if err := types.ConfigureLogger(clog); err != nil {
  419. log.Fatalf("While creating bucket-specific logger : %s", err)
  420. }
  421. clog.SetLevel(log.DebugLevel)
  422. n.Logger = clog.WithFields(log.Fields{
  423. "id": n.rn,
  424. })
  425. n.Logger.Infof("%s has debug enabled", n.Name)
  426. } else {
  427. /* else bind it to the default one (might find something more elegant here)*/
  428. n.Logger = log.WithFields(log.Fields{
  429. "id": n.rn,
  430. })
  431. }
  432. /* display info about top-level nodes, they should be the only one with explicit stage name ?*/
  433. n.Logger = n.Logger.WithFields(log.Fields{"stage": n.Stage, "name": n.Name})
  434. n.Logger.Tracef("Compiling : %s", dumpr.Sdump(n))
  435. //compile filter if present
  436. if n.Filter != "" {
  437. n.RunTimeFilter, err = expr.Compile(n.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  438. if err != nil {
  439. return fmt.Errorf("compilation of '%s' failed: %v", n.Filter, err)
  440. }
  441. if n.Debug {
  442. n.ExprDebugger, err = exprhelpers.NewDebugger(n.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  443. if err != nil {
  444. log.Errorf("unable to build debug filter for '%s' : %s", n.Filter, err)
  445. }
  446. }
  447. }
  448. /* handle pattern_syntax and groks */
  449. for _, pattern := range n.SubGroks {
  450. n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value)
  451. if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil {
  452. if errors.Is(err, grokky.ErrAlreadyExist) {
  453. n.Logger.Warningf("grok '%s' already registred", pattern.Key)
  454. continue
  455. }
  456. n.Logger.Errorf("Unable to compile subpattern %s : %v", pattern.Key, err)
  457. return err
  458. }
  459. }
  460. /* load grok by name or compile in-place */
  461. if n.Grok.RegexpName != "" {
  462. n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)
  463. n.Grok.RunTimeRegexp, err = pctx.Grok.Get(n.Grok.RegexpName)
  464. if err != nil {
  465. return fmt.Errorf("unable to find grok '%s' : %v", n.Grok.RegexpName, err)
  466. }
  467. if n.Grok.RunTimeRegexp == nil {
  468. return fmt.Errorf("empty grok '%s'", n.Grok.RegexpName)
  469. }
  470. n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpName, n.Grok.RunTimeRegexp.Regexp.String())
  471. valid = true
  472. } else if n.Grok.RegexpValue != "" {
  473. if strings.HasSuffix(n.Grok.RegexpValue, "\n") {
  474. n.Logger.Debugf("Beware, pattern ends with \\n : '%s'", n.Grok.RegexpValue)
  475. }
  476. n.Grok.RunTimeRegexp, err = pctx.Grok.Compile(n.Grok.RegexpValue)
  477. if err != nil {
  478. return fmt.Errorf("failed to compile grok '%s': %v", n.Grok.RegexpValue, err)
  479. }
  480. if n.Grok.RunTimeRegexp == nil {
  481. // We shouldn't be here because compilation succeeded, so regexp shouldn't be nil
  482. return fmt.Errorf("grok compilation failure: %s", n.Grok.RegexpValue)
  483. }
  484. n.Logger.Tracef("%s regexp : %s", n.Grok.RegexpValue, n.Grok.RunTimeRegexp.Regexp.String())
  485. valid = true
  486. }
  487. /*if grok source is an expression*/
  488. if n.Grok.ExpValue != "" {
  489. n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue,
  490. expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  491. if err != nil {
  492. return errors.Wrap(err, "while compiling grok's expression")
  493. }
  494. }
  495. /* load grok statics */
  496. if len(n.Grok.Statics) > 0 {
  497. //compile expr statics if present
  498. for idx := range n.Grok.Statics {
  499. if n.Grok.Statics[idx].ExpValue != "" {
  500. n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue,
  501. expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  502. if err != nil {
  503. return err
  504. }
  505. }
  506. }
  507. valid = true
  508. }
  509. /* load data capture (stash) */
  510. for i, stash := range n.Stash {
  511. n.Stash[i].ValueExpression, err = expr.Compile(stash.Value,
  512. expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  513. if err != nil {
  514. return errors.Wrap(err, "while compiling stash value expression")
  515. }
  516. n.Stash[i].KeyExpression, err = expr.Compile(stash.Key,
  517. expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  518. if err != nil {
  519. return errors.Wrap(err, "while compiling stash key expression")
  520. }
  521. n.Stash[i].TTLVal, err = time.ParseDuration(stash.TTL)
  522. if err != nil {
  523. return errors.Wrap(err, "while parsing stash ttl")
  524. }
  525. logLvl := n.Logger.Logger.GetLevel()
  526. //init the cache, does it make sense to create it here just to be sure everything is fine ?
  527. if err := cache.CacheInit(cache.CacheCfg{
  528. Size: n.Stash[i].MaxMapSize,
  529. TTL: n.Stash[i].TTLVal,
  530. Name: n.Stash[i].Name,
  531. LogLevel: &logLvl,
  532. }); err != nil {
  533. return errors.Wrap(err, "while initializing cache")
  534. }
  535. }
  536. /* compile leafs if present */
  537. if len(n.LeavesNodes) > 0 {
  538. for idx := range n.LeavesNodes {
  539. if n.LeavesNodes[idx].Name == "" {
  540. n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name)
  541. }
  542. /*propagate debug/stats to child nodes*/
  543. if !n.LeavesNodes[idx].Debug && n.Debug {
  544. n.LeavesNodes[idx].Debug = true
  545. }
  546. if !n.LeavesNodes[idx].Profiling && n.Profiling {
  547. n.LeavesNodes[idx].Profiling = true
  548. }
  549. n.LeavesNodes[idx].Stage = n.Stage
  550. err = n.LeavesNodes[idx].compile(pctx, ectx)
  551. if err != nil {
  552. return err
  553. }
  554. }
  555. valid = true
  556. }
  557. /* load statics if present */
  558. for idx := range n.Statics {
  559. if n.Statics[idx].ExpValue != "" {
  560. n.Statics[idx].RunTimeValue, err = expr.Compile(n.Statics[idx].ExpValue, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  561. if err != nil {
  562. n.Logger.Errorf("Statics Compilation failed %v.", err)
  563. return err
  564. }
  565. }
  566. valid = true
  567. }
  568. /* compile whitelists if present */
  569. for _, v := range n.Whitelist.Ips {
  570. n.Whitelist.B_Ips = append(n.Whitelist.B_Ips, net.ParseIP(v))
  571. n.Logger.Debugf("adding ip %s to whitelists", net.ParseIP(v))
  572. valid = true
  573. }
  574. for _, v := range n.Whitelist.Cidrs {
  575. _, tnet, err := net.ParseCIDR(v)
  576. if err != nil {
  577. n.Logger.Fatalf("Unable to parse cidr whitelist '%s' : %v.", v, err)
  578. }
  579. n.Whitelist.B_Cidrs = append(n.Whitelist.B_Cidrs, tnet)
  580. n.Logger.Debugf("adding cidr %s to whitelists", tnet)
  581. valid = true
  582. }
  583. for _, filter := range n.Whitelist.Exprs {
  584. expression := &ExprWhitelist{}
  585. expression.Filter, err = expr.Compile(filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  586. if err != nil {
  587. n.Logger.Fatalf("Unable to compile whitelist expression '%s' : %v.", filter, err)
  588. }
  589. expression.ExprDebugger, err = exprhelpers.NewDebugger(filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
  590. if err != nil {
  591. log.Errorf("unable to build debug filter for '%s' : %s", filter, err)
  592. }
  593. n.Whitelist.B_Exprs = append(n.Whitelist.B_Exprs, expression)
  594. n.Logger.Debugf("adding expression %s to whitelists", filter)
  595. valid = true
  596. }
  597. if !valid {
  598. /* node is empty, error force return */
  599. n.Logger.Error("Node is empty or invalid, abort")
  600. n.Stage = ""
  601. return fmt.Errorf("Node is empty")
  602. }
  603. if err := n.validate(pctx, ectx); err != nil {
  604. return err
  605. }
  606. return nil
  607. }