unix_parser.go 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. package parser
  2. import (
  3. "fmt"
  4. "os"
  5. "path/filepath"
  6. "sort"
  7. "strings"
  8. log "github.com/sirupsen/logrus"
  9. "github.com/crowdsecurity/grokky"
  10. "github.com/crowdsecurity/crowdsec/pkg/csconfig"
  11. "github.com/crowdsecurity/crowdsec/pkg/cwhub"
  12. "github.com/crowdsecurity/crowdsec/pkg/fflag"
  13. )
  14. type UnixParserCtx struct {
  15. Grok grokky.Host
  16. Stages []string
  17. Profiling bool
  18. DataFolder string
  19. }
  20. type Parsers struct {
  21. Ctx *UnixParserCtx
  22. Povfwctx *UnixParserCtx
  23. StageFiles []Stagefile
  24. PovfwStageFiles []Stagefile
  25. Nodes []Node
  26. Povfwnodes []Node
  27. EnricherCtx EnricherCtx
  28. }
  29. func Init(c map[string]interface{}) (*UnixParserCtx, error) {
  30. r := UnixParserCtx{}
  31. r.Grok = grokky.NewBase()
  32. r.Grok.UseRe2 = fflag.Re2GrokSupport.IsEnabled()
  33. files, err := os.ReadDir(c["patterns"].(string))
  34. if err != nil {
  35. return nil, err
  36. }
  37. r.DataFolder = c["data"].(string)
  38. for _, f := range files {
  39. if strings.Contains(f.Name(), ".") {
  40. continue
  41. }
  42. if err := r.Grok.AddFromFile(filepath.Join(c["patterns"].(string), f.Name())); err != nil {
  43. log.Errorf("failed to load pattern %s : %v", f.Name(), err)
  44. return nil, err
  45. }
  46. }
  47. log.Debugf("Loaded %d pattern files", len(files))
  48. return &r, nil
  49. }
  50. // Return new parsers
  51. // nodes and povfwnodes are already initialized in parser.LoadStages
  52. func NewParsers() *Parsers {
  53. parsers := &Parsers{
  54. Ctx: &UnixParserCtx{},
  55. Povfwctx: &UnixParserCtx{},
  56. StageFiles: make([]Stagefile, 0),
  57. PovfwStageFiles: make([]Stagefile, 0),
  58. }
  59. for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
  60. for _, hubParserItem := range cwhub.GetItemMap(itemType) {
  61. if hubParserItem.Installed {
  62. stagefile := Stagefile{
  63. Filename: hubParserItem.LocalPath,
  64. Stage: hubParserItem.Stage,
  65. }
  66. if itemType == cwhub.PARSERS {
  67. parsers.StageFiles = append(parsers.StageFiles, stagefile)
  68. }
  69. if itemType == cwhub.PARSERS_OVFLW {
  70. parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
  71. }
  72. }
  73. }
  74. }
  75. if parsers.StageFiles != nil {
  76. sort.Slice(parsers.StageFiles, func(i, j int) bool {
  77. return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
  78. })
  79. }
  80. if parsers.PovfwStageFiles != nil {
  81. sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
  82. return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
  83. })
  84. }
  85. return parsers
  86. }
  87. func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
  88. var err error
  89. patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
  90. log.Infof("Loading grok library %s", patternsDir)
  91. /* load base regexps for two grok parsers */
  92. parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
  93. "data": cConfig.Crowdsec.DataDir})
  94. if err != nil {
  95. return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
  96. }
  97. parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
  98. "data": cConfig.Crowdsec.DataDir})
  99. if err != nil {
  100. return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
  101. }
  102. /*
  103. Load enrichers
  104. */
  105. log.Infof("Loading enrich plugins")
  106. parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
  107. if err != nil {
  108. return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
  109. }
  110. /*
  111. Load the actual parsers
  112. */
  113. log.Infof("Loading parsers from %d files", len(parsers.StageFiles))
  114. parsers.Nodes, err = LoadStages(parsers.StageFiles, parsers.Ctx, parsers.EnricherCtx)
  115. if err != nil {
  116. return parsers, fmt.Errorf("failed to load parser config : %v", err)
  117. }
  118. if len(parsers.PovfwStageFiles) > 0 {
  119. log.Infof("Loading postoverflow parsers")
  120. parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx)
  121. } else {
  122. log.Infof("No postoverflow parsers to load")
  123. parsers.Povfwnodes = []Node{}
  124. }
  125. if err != nil {
  126. return parsers, fmt.Errorf("failed to load postoverflow config : %v", err)
  127. }
  128. if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
  129. parsers.Ctx.Profiling = true
  130. parsers.Povfwctx.Profiling = true
  131. }
  132. return parsers, nil
  133. }