hub ci attempt
This commit is contained in:
Thibault bui Koechlin
parent
0b5cd18533
commit
7aed35c4b6
1 changed files with 204 additions and 0 deletions
204
cmd/hub-ci/main.go
Normal file
204
cmd/hub-ci/main.go
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gopkg.in/tomb.v2"
|
||||
)
|
||||
|
||||
var acquisTomb tomb.Tomb
|
||||
var testDir string
|
||||
|
||||
var AllResults []LineParseResult
|
||||
var AllExpected []LineParseResult
|
||||
|
||||
type LineParseResult struct {
|
||||
Line string
|
||||
ParserResults map[string]map[string]types.Event
|
||||
}
|
||||
|
||||
//cleanForMatch : cleanup results from items that might change every run
|
||||
func cleanForMatch(in map[string]map[string]types.Event) map[string]map[string]types.Event {
|
||||
for stage, val := range in {
|
||||
for parser, evt := range val {
|
||||
evt.Line.Time = time.Time{}
|
||||
in[stage][parser] = evt
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func parseMatchLine(event types.Event, parserCTX *parser.UnixParserCtx, parserNodes []parser.Node) bool {
|
||||
oneResult := LineParseResult{}
|
||||
h := sha256.New()
|
||||
h.Write([]byte(event.Line.Raw))
|
||||
log.Printf("processing '%s'", event.Line.Raw)
|
||||
|
||||
//parse
|
||||
parsed, err := parser.Parse(*parserCTX, event, parserNodes)
|
||||
if err != nil {
|
||||
log.Fatalf("failed parsing : %v\n", err)
|
||||
}
|
||||
|
||||
if !parsed.Process {
|
||||
log.Warningf("Unparsed: %s", parsed.Line.Raw)
|
||||
}
|
||||
//marshal current result
|
||||
oneResult.Line = parsed.Line.Raw
|
||||
//we need to clean Line's timestamp
|
||||
oneResult.ParserResults = cleanForMatch(parser.StageParseCache)
|
||||
/*
|
||||
** we are using cmp's feature to match structures.
|
||||
** because of the way marshal/unmarshal works we want to make nil == empty
|
||||
*/
|
||||
// This option handles slices and maps of any type.
|
||||
alwaysEqual := cmp.Comparer(func(_, _ interface{}) bool { return true })
|
||||
opt := cmp.FilterValues(func(x, y interface{}) bool {
|
||||
vx, vy := reflect.ValueOf(x), reflect.ValueOf(y)
|
||||
return (vx.IsValid() && vy.IsValid() && vx.Type() == vy.Type()) &&
|
||||
(vx.Kind() == reflect.Slice || vx.Kind() == reflect.Map) &&
|
||||
(vx.Len() == 0 && vy.Len() == 0)
|
||||
}, alwaysEqual)
|
||||
|
||||
/*
|
||||
Iterate over the list of expected results and try to find back
|
||||
*/
|
||||
AllResults = append(AllResults, oneResult)
|
||||
matched := false
|
||||
for idx, candidate := range AllExpected {
|
||||
//not our line
|
||||
if candidate.Line != event.Line.Raw {
|
||||
continue
|
||||
}
|
||||
if cmp.Equal(candidate, oneResult, opt) {
|
||||
matched = true
|
||||
//we go an exact match
|
||||
log.Printf("Found exact match (idx:%d)", idx)
|
||||
//cleanup
|
||||
AllExpected = append(AllExpected[:idx], AllExpected[idx+1:]...)
|
||||
} else {
|
||||
log.Printf("Mismatch for line :")
|
||||
log.Printf("%s", cmp.Diff(candidate, oneResult, opt))
|
||||
}
|
||||
break
|
||||
}
|
||||
if !matched && len(AllExpected) != 0 {
|
||||
log.Fatalf("Result is not in the %d expected results", len(AllExpected))
|
||||
}
|
||||
return matched
|
||||
}
|
||||
|
||||
func main() {
|
||||
var (
|
||||
err error
|
||||
p parser.UnixParser
|
||||
parserCTX *parser.UnixParserCtx
|
||||
parserNodes []parser.Node = make([]parser.Node, 0)
|
||||
acquisitionCTX *acquisition.FileAcquisCtx
|
||||
cConfig *csconfig.CrowdSec
|
||||
)
|
||||
inputLineChan := make(chan types.Event)
|
||||
log.SetLevel(log.InfoLevel)
|
||||
cConfig = csconfig.NewCrowdSecConfig()
|
||||
|
||||
test_dir := os.Args[1]
|
||||
|
||||
cConfig.AcquisitionFile = test_dir + "/acquis.yaml"
|
||||
log.Printf("Setting acquis source to %s", cConfig.AcquisitionFile)
|
||||
/* load base regexps for two grok parsers */
|
||||
parserCTX, err = p.Init(map[string]interface{}{"patterns": cConfig.ConfigFolder + string("/patterns/"), "data": cConfig.DataFolder})
|
||||
if err != nil {
|
||||
log.Errorf("failed to initialize parser : %v", err)
|
||||
return
|
||||
}
|
||||
/* Load enrichers */
|
||||
log.Infof("Loading enrich plugins")
|
||||
parserPlugins, err := parser.Loadplugin(cConfig.DataFolder)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to load plugin geoip : %v", err)
|
||||
}
|
||||
parser.ECTX = append(parser.ECTX, parserPlugins)
|
||||
//load parsers
|
||||
log.Infof("Loading parsers")
|
||||
parserNodes, err = parser.LoadStageDir(cConfig.ConfigFolder+"/parsers/", parserCTX)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to load parser config : %v", err)
|
||||
}
|
||||
|
||||
//Init the acqusition : from cli or from acquis.yaml file
|
||||
acquisitionCTX, err = acquisition.LoadAcquisitionConfig(cConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to start acquisition : %s", err)
|
||||
}
|
||||
|
||||
if len(acquisitionCTX.Files) != 1 {
|
||||
log.Fatalf("only one file per dir")
|
||||
}
|
||||
|
||||
//load the expected results
|
||||
ExpectedPresent := false
|
||||
expectedResultsFile := test_dir + "/results.yaml"
|
||||
expected_bytes, err := ioutil.ReadFile(expectedResultsFile)
|
||||
if err != nil {
|
||||
log.Warningf("no results in %s, will dump data instead!", test_dir)
|
||||
} else {
|
||||
if err := json.Unmarshal(expected_bytes, &AllExpected); err != nil {
|
||||
log.Fatalf("file %s can't be unmarshaled : %s", expectedResultsFile, err)
|
||||
} else {
|
||||
ExpectedPresent = true
|
||||
}
|
||||
}
|
||||
|
||||
//start reading in the background
|
||||
acquisition.AcquisStartReading(acquisitionCTX, inputLineChan, &acquisTomb)
|
||||
|
||||
//Try to load the results file
|
||||
//expected_bytes, err := ioutil.ReadFile(test_dir)
|
||||
|
||||
go func() {
|
||||
log.Printf("starting to process stuff!")
|
||||
parser.ParseDump = true
|
||||
for event := range inputLineChan {
|
||||
if !parseMatchLine(event, parserCTX, parserNodes) {
|
||||
fmt.Printf("while parsing:\n%s\n", event.Line.Raw)
|
||||
//log.Fatalf("mismatch test")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
if err := acquisTomb.Wait(); err != nil {
|
||||
log.Warningf("acquisition returned error : %s", err)
|
||||
}
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
/*now let's check the results*/
|
||||
|
||||
//there was no data present, just dump
|
||||
if !ExpectedPresent {
|
||||
log.Warningf("No expected results loaded, dump.")
|
||||
dump_bytes, err := json.MarshalIndent(AllResults, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("failed to marshal results : %s", err)
|
||||
}
|
||||
if err := ioutil.WriteFile(expectedResultsFile, dump_bytes, 0644); err != nil {
|
||||
log.Fatalf("failed to dump data to %s : %s", expectedResultsFile, err)
|
||||
}
|
||||
} else {
|
||||
if len(AllExpected) > 0 {
|
||||
log.Errorf("Left-over results in expected : %d", len(AllExpected))
|
||||
}
|
||||
}
|
||||
log.Infof("tests are finished.")
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue