unix_parser.go 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141
  1. package parser
  2. import (
  3. "fmt"
  4. "io/ioutil"
  5. "path"
  6. "sort"
  7. "github.com/crowdsecurity/crowdsec/pkg/csconfig"
  8. "github.com/crowdsecurity/crowdsec/pkg/cwhub"
  9. "github.com/crowdsecurity/grokky"
  10. log "github.com/sirupsen/logrus"
  11. )
  12. type UnixParserCtx struct {
  13. Grok grokky.Host
  14. Stages []string
  15. Profiling bool
  16. DataFolder string
  17. }
  18. type Parsers struct {
  19. Ctx *UnixParserCtx
  20. Povfwctx *UnixParserCtx
  21. StageFiles []Stagefile
  22. PovfwStageFiles []Stagefile
  23. Nodes []Node
  24. Povfwnodes []Node
  25. EnricherCtx EnricherCtx
  26. }
  27. func Init(c map[string]interface{}) (*UnixParserCtx, error) {
  28. r := UnixParserCtx{}
  29. r.Grok = grokky.NewBase()
  30. files, err := ioutil.ReadDir(c["patterns"].(string))
  31. if err != nil {
  32. return nil, err
  33. }
  34. r.DataFolder = c["data"].(string)
  35. for _, f := range files {
  36. if err := r.Grok.AddFromFile(path.Join(c["patterns"].(string), f.Name())); err != nil {
  37. log.Errorf("failed to load pattern %s : %v", f.Name(), err)
  38. return nil, err
  39. }
  40. }
  41. log.Debugf("Loaded %d pattern files", len(files))
  42. return &r, nil
  43. }
  44. // Return new parsers
  45. // nodes and povfwnodes are already initialized in LoadStages
  46. func NewParsers() *Parsers {
  47. parsers := &Parsers{
  48. Ctx: &UnixParserCtx{},
  49. Povfwctx: &UnixParserCtx{},
  50. StageFiles: make([]Stagefile, 0),
  51. PovfwStageFiles: make([]Stagefile, 0),
  52. }
  53. for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
  54. for _, hubParserItem := range cwhub.GetItemMap(itemType) {
  55. if hubParserItem.Installed {
  56. stagefile := Stagefile{
  57. Filename: hubParserItem.LocalPath,
  58. Stage: hubParserItem.Stage,
  59. }
  60. if itemType == cwhub.PARSERS {
  61. parsers.StageFiles = append(parsers.StageFiles, stagefile)
  62. }
  63. if itemType == cwhub.PARSERS_OVFLW {
  64. parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
  65. }
  66. }
  67. }
  68. }
  69. if parsers.StageFiles != nil {
  70. sort.Slice(parsers.StageFiles, func(i, j int) bool {
  71. return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
  72. })
  73. }
  74. if parsers.PovfwStageFiles != nil {
  75. sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
  76. return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
  77. })
  78. }
  79. return parsers
  80. }
  81. func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
  82. var err error
  83. patternsDir := path.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
  84. log.Infof("Loading grok library %s", patternsDir)
  85. /* load base regexps for two grok parsers */
  86. parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
  87. "data": cConfig.Crowdsec.DataDir})
  88. if err != nil {
  89. return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
  90. }
  91. parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
  92. "data": cConfig.Crowdsec.DataDir})
  93. if err != nil {
  94. return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
  95. }
  96. /*
  97. Load enrichers
  98. */
  99. log.Infof("Loading enrich plugins")
  100. parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
  101. if err != nil {
  102. return parsers, fmt.Errorf("Failed to load enrich plugin : %v", err)
  103. }
  104. /*
  105. Load the actual parsers
  106. */
  107. log.Infof("Loading parsers %d stages", len(parsers.StageFiles))
  108. parsers.Nodes, err = LoadStages(parsers.StageFiles, parsers.Ctx, parsers.EnricherCtx)
  109. if err != nil {
  110. return parsers, fmt.Errorf("failed to load parser config : %v", err)
  111. }
  112. log.Infof("Loading postoverflow Parsers")
  113. parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx)
  114. if err != nil {
  115. return parsers, fmt.Errorf("failed to load postoverflow config : %v", err)
  116. }
  117. if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
  118. parsers.Ctx.Profiling = true
  119. parsers.Povfwctx.Profiling = true
  120. }
  121. return parsers, nil
  122. }