unix_parser.go 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150
  1. package parser
  2. import (
  3. "fmt"
  4. "os"
  5. "path"
  6. "sort"
  7. "strings"
  8. "github.com/crowdsecurity/crowdsec/pkg/csconfig"
  9. "github.com/crowdsecurity/crowdsec/pkg/cwhub"
  10. "github.com/crowdsecurity/grokky"
  11. log "github.com/sirupsen/logrus"
  12. )
  13. type UnixParserCtx struct {
  14. Grok grokky.Host
  15. Stages []string
  16. Profiling bool
  17. DataFolder string
  18. }
  19. type Parsers struct {
  20. Ctx *UnixParserCtx
  21. Povfwctx *UnixParserCtx
  22. StageFiles []Stagefile
  23. PovfwStageFiles []Stagefile
  24. Nodes []Node
  25. Povfwnodes []Node
  26. EnricherCtx EnricherCtx
  27. }
  28. func Init(c map[string]interface{}) (*UnixParserCtx, error) {
  29. r := UnixParserCtx{}
  30. r.Grok = grokky.NewBase()
  31. files, err := os.ReadDir(c["patterns"].(string))
  32. if err != nil {
  33. return nil, err
  34. }
  35. r.DataFolder = c["data"].(string)
  36. for _, f := range files {
  37. if strings.Contains(f.Name(), ".") {
  38. continue
  39. }
  40. if err := r.Grok.AddFromFile(path.Join(c["patterns"].(string), f.Name())); err != nil {
  41. log.Errorf("failed to load pattern %s : %v", f.Name(), err)
  42. return nil, err
  43. }
  44. }
  45. log.Debugf("Loaded %d pattern files", len(files))
  46. return &r, nil
  47. }
  48. // Return new parsers
  49. // nodes and povfwnodes are already initialized in parser.LoadStages
  50. func NewParsers() *Parsers {
  51. parsers := &Parsers{
  52. Ctx: &UnixParserCtx{},
  53. Povfwctx: &UnixParserCtx{},
  54. StageFiles: make([]Stagefile, 0),
  55. PovfwStageFiles: make([]Stagefile, 0),
  56. }
  57. for _, itemType := range []string{cwhub.PARSERS, cwhub.PARSERS_OVFLW} {
  58. for _, hubParserItem := range cwhub.GetItemMap(itemType) {
  59. if hubParserItem.Installed {
  60. stagefile := Stagefile{
  61. Filename: hubParserItem.LocalPath,
  62. Stage: hubParserItem.Stage,
  63. }
  64. if itemType == cwhub.PARSERS {
  65. parsers.StageFiles = append(parsers.StageFiles, stagefile)
  66. }
  67. if itemType == cwhub.PARSERS_OVFLW {
  68. parsers.PovfwStageFiles = append(parsers.PovfwStageFiles, stagefile)
  69. }
  70. }
  71. }
  72. }
  73. if parsers.StageFiles != nil {
  74. sort.Slice(parsers.StageFiles, func(i, j int) bool {
  75. return parsers.StageFiles[i].Filename < parsers.StageFiles[j].Filename
  76. })
  77. }
  78. if parsers.PovfwStageFiles != nil {
  79. sort.Slice(parsers.PovfwStageFiles, func(i, j int) bool {
  80. return parsers.PovfwStageFiles[i].Filename < parsers.PovfwStageFiles[j].Filename
  81. })
  82. }
  83. return parsers
  84. }
  85. func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
  86. var err error
  87. patternsDir := path.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
  88. log.Infof("Loading grok library %s", patternsDir)
  89. /* load base regexps for two grok parsers */
  90. parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
  91. "data": cConfig.Crowdsec.DataDir})
  92. if err != nil {
  93. return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
  94. }
  95. parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
  96. "data": cConfig.Crowdsec.DataDir})
  97. if err != nil {
  98. return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
  99. }
  100. /*
  101. Load enrichers
  102. */
  103. log.Infof("Loading enrich plugins")
  104. parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
  105. if err != nil {
  106. return parsers, fmt.Errorf("Failed to load enrich plugin : %v", err)
  107. }
  108. /*
  109. Load the actual parsers
  110. */
  111. log.Infof("Loading parsers from %d files", len(parsers.StageFiles))
  112. parsers.Nodes, err = LoadStages(parsers.StageFiles, parsers.Ctx, parsers.EnricherCtx)
  113. if err != nil {
  114. return parsers, fmt.Errorf("failed to load parser config : %v", err)
  115. }
  116. if len(parsers.PovfwStageFiles) > 0 {
  117. log.Infof("Loading postoverflow parsers")
  118. parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx)
  119. } else {
  120. parsers.Povfwnodes = []Node{}
  121. log.Infof("No postoverflow parsers to load")
  122. }
  123. if err != nil {
  124. return parsers, fmt.Errorf("failed to load postoverflow config : %v", err)
  125. }
  126. if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
  127. parsers.Ctx.Profiling = true
  128. parsers.Povfwctx.Profiling = true
  129. }
  130. return parsers, nil
  131. }