wip

cmd/crowdsec/main.go

@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"strings"
 
 	_ "net/http/pprof"
 	"time"
@@ -18,6 +17,7 @@ import (
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/pkg/errors"
 
 	log "github.com/sirupsen/logrus"
 
@@ -46,18 +46,16 @@ var (
 )
 
 type Flags struct {
-	ConfigFile             string
+	ConfigFile     string
-	TraceLevel             bool
+	TraceLevel     bool
-	DebugLevel             bool
+	DebugLevel     bool
-	InfoLevel              bool
+	InfoLevel      bool
-	PrintVersion           bool
+	PrintVersion   bool
-	SingleFilePath         string
+	SingleFileType string
-	SingleJournalctlFilter string
+	OneShotDSN     string
-	SingleFileType         string
+	TestMode       bool
-	SingleFileJsonOutput   string
+	DisableAgent   bool
-	TestMode               bool
+	DisableAPI     bool
-	DisableAgent           bool
-	DisableAPI             bool
 }
 
 type parsers struct {
@@ -140,30 +138,19 @@ func LoadBuckets(cConfig *csconfig.Config) error {
 func LoadAcquisition(cConfig *csconfig.Config) error {
 	var err error
 
-	if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
+	if flags.SingleFileType != "" || flags.OneShotDSN != "" {
-
+		if flags.OneShotDSN == "" || flags.SingleFileType == "" {
-		tmpCfg := acquisition.DataSourceCfg{}
+			return fmt.Errorf("-type requires a -dsn argument")
-		tmpCfg.Mode = acquisition.CAT_MODE
-		tmpCfg.Labels = map[string]string{"type": flags.SingleFileType}
-
-		if flags.SingleFilePath != "" {
-			tmpCfg.Filename = flags.SingleFilePath
-		} else if flags.SingleJournalctlFilter != "" {
-			tmpCfg.JournalctlFilters = strings.Split(flags.SingleJournalctlFilter, " ")
 		}
 
-		datasrc, err := acquisition.DataSourceConfigure(tmpCfg)
+		dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.SingleFileType)
 		if err != nil {
-			return fmt.Errorf("while configuring specified file datasource : %s", err)
+			return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN)
 		}
-		if dataSources == nil {
-			dataSources = make([]acquisition.DataSource, 0)
-		}
-		dataSources = append(dataSources, datasrc)
 	} else {
 		dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
 		if err != nil {
-			log.Fatalf("While loading acquisition configuration : %s", err)
+			return errors.Wrap(err, "while loading acquisition configuration")
 		}
 	}
 
@@ -177,8 +164,7 @@ func (f *Flags) Parse() {
 	flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stdout")
 	flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stdout")
 	flag.BoolVar(&f.PrintVersion, "version", false, "display version")
-	flag.StringVar(&f.SingleFilePath, "file", "", "Process a single file in time-machine")
+	flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine")
-	flag.StringVar(&f.SingleJournalctlFilter, "jfilter", "", "Process a single journalctl output in time-machine")
 	flag.StringVar(&f.SingleFileType, "type", "", "Labels.type for file in time-machine")
 	flag.BoolVar(&f.TestMode, "t", false, "only test configs")
 	flag.BoolVar(&f.DisableAgent, "no-cs", false, "disable crowdsec agent")
@@ -210,17 +196,17 @@ func LoadConfig(cConfig *csconfig.Config) error {
 		log.Fatalf("You must run at least the API Server or crowdsec")
 	}
 
-	if flags.SingleFilePath != "" {
+	// if flags.SingleFilePath != "" {
-		if flags.SingleFileType == "" {
+	// 	if flags.SingleFileType == "" {
-			return fmt.Errorf("-file requires -type")
+	// 		return fmt.Errorf("-file requires -type")
-		}
+	// 	}
-	}
+	// }
 
-	if flags.SingleJournalctlFilter != "" {
+	// if flags.SingleJournalctlFilter != "" {
-		if flags.SingleFileType == "" {
+	// 	if flags.SingleFileType == "" {
-			return fmt.Errorf("-jfilter requires -type")
+	// 		return fmt.Errorf("-jfilter requires -type")
-		}
+	// 	}
-	}
+	// }
 
 	if flags.DebugLevel {
 		logLevel := log.DebugLevel
@@ -239,7 +225,7 @@ func LoadConfig(cConfig *csconfig.Config) error {
 		cConfig.Crowdsec.LintOnly = true
 	}
 
-	if flags.SingleFilePath != "" || flags.SingleJournalctlFilter != "" {
+	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
 		cConfig.API.Server.OnlineClient = nil
 		/*if the api is disabled as well, just read file and exit, don't daemonize*/
 		if flags.DisableAPI {

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/docker/docker v20.10.2+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/enescakir/emoji v1.0.0
+	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-co-op/gocron v0.5.1
 	github.com/go-openapi/errors v0.19.9
@@ -47,21 +48,19 @@ require (
 	github.com/oschwald/geoip2-golang v1.4.0
 	github.com/oschwald/maxminddb-golang v1.8.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.9.0
+	github.com/prometheus/client_golang v1.10.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/procfs v0.3.0 // indirect
 	github.com/prometheus/prom2json v1.3.0
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sirupsen/logrus v1.7.0
+	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.1.3
 	github.com/stretchr/testify v1.7.0
 	github.com/ugorji/go v1.2.3 // indirect
 	github.com/vjeantet/grok v1.0.1 // indirect
 	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
 	golang.org/x/mod v0.4.1
-	golang.org/x/net v0.0.0-20201224014010-6772e930b67b
+	golang.org/x/net v0.0.0-20201224014010-6772e930b67b // indirect
-	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
 	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
 	golang.org/x/text v0.3.5 // indirect
 	google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f // indirect
@@ -69,7 +68,7 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
 	gopkg.in/yaml.v2 v2.4.0
-	gotest.tools/v3 v3.0.3 // indirect
+	gotest.tools/v3 v3.0.3
 )
 
 replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0

go.sum

@@ -40,11 +40,9 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antonmedv/expr v1.8.9 h1:O9stiHmHHww9b4ozhPx7T6BK7fXfOCHJ8ybxf0833zw=
@@ -62,7 +60,6 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
 github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg=
 github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
@@ -98,7 +95,6 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -147,7 +143,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
-github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -160,7 +155,6 @@ github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
-github.com/go-bindata/go-bindata v1.0.1-0.20190711162640-ee3c2418e368 h1:WNHfSP1q2vuAa9vF54RrhCl4nqxCjVcXhlbsRXbGOSY=
 github.com/go-bindata/go-bindata v1.0.1-0.20190711162640-ee3c2418e368/go.mod h1:7xCgX1lzlrXPHkfvn3EhumqHkmSlzt8at9q7v0ax19c=
 github.com/go-co-op/gocron v0.5.1 h1:Cni1V7mt184+HnYTDYe6MH7siofCvf94PrGyIDI1v1U=
 github.com/go-co-op/gocron v0.5.1/go.mod h1:6Btk4lVj3bnFAgbVfr76W8impTyhYrEi1pV5Pt4Tp/M=
@@ -177,7 +171,6 @@ github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpR
 github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
 github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
 github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
-github.com/go-openapi/analysis v0.19.10 h1:5BHISBAXOc/aJK25irLZnx2D3s6WyYaY9D4gmuz9fdE=
 github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ=
 github.com/go-openapi/analysis v0.19.16 h1:Ub9e++M8sDwtHD+S587TYi+6ANBG1NRYGZDihqk0SaY=
 github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk=
@@ -186,25 +179,20 @@ github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQH
 github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
 github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
 github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.19.7 h1:Lcq+o0mSwCLKACMxZhreVHigB9ebghJ/lrmeaqASbjo=
 github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/errors v0.19.8 h1:doM+tQdZbUm9gydV9yR+iQNmztbjj7I3sW4sIcAwIzc=
 github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
 github.com/go-openapi/errors v0.19.9 h1:9SnKdGhiPZHF3ttwFMiCBEb8jQ4IDdrK+5+a0oTygA4=
 github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M=
-github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4=
 github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
-github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
-github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
 github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
 github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
@@ -213,9 +201,7 @@ github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf
 github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
 github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
 github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI=
-github.com/go-openapi/loads v0.19.5 h1:jZVYWawIQiA1NBnHla28ktg6hrcfTHsCE+3QLVRBIls=
 github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY=
-github.com/go-openapi/loads v0.19.6 h1:6IAtnx22MNSjPocZZ2sV7EjgF6wW5rDC9r6ZkNxjiN8=
 github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
 github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc=
 github.com/go-openapi/loads v0.20.0 h1:Pymw1O8zDmWeNv4kVsHd0W3cvgdp8juRa4U/U/8D/Pk=
@@ -224,7 +210,6 @@ github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6
 github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
 github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
 github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo=
-github.com/go-openapi/runtime v0.19.16 h1:tQMAY5s5BfmmCC31+ufDCsGrr8iO1A8UIdYfDo5ADvs=
 github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98=
 github.com/go-openapi/runtime v0.19.24 h1:TqagMVlRAOTwllE/7hNKx6rQ10O6T8ZzeJdMjSTKaD4=
 github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk=
@@ -233,7 +218,6 @@ github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsd
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
-github.com/go-openapi/spec v0.19.8 h1:qAdZLh1r6QF/hI/gTq+TJTvsQUodZsM7KLqkAJdiJNg=
 github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
 github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU=
 github.com/go-openapi/spec v0.20.0 h1:HGLc8AJ7ynOxwv0Lq4TsnwLsWMawHAYiJIFzbcML86I=
@@ -244,7 +228,6 @@ github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+Z
 github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
 github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
 github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
-github.com/go-openapi/strfmt v0.19.5 h1:0utjKrw+BAh8s57XE9Xz8DUBsVvPmRUB6styvl9wWIM=
 github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
 github.com/go-openapi/strfmt v0.19.11 h1:0+YvbNh05rmBkgztd6zHp4OCFn7Mtu30bn46NQo2ZRw=
 github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc=
@@ -253,7 +236,6 @@ github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
-github.com/go-openapi/swag v0.19.9 h1:1IxuqvBUU3S2Bi4YC7tlP9SJF1gVpCvqN0T2Qof4azE=
 github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY=
 github.com/go-openapi/swag v0.19.12 h1:Bc0bnY2c3AoF7Gc+IMIAQQsD8fLHjHpc19wXvYuayQI=
 github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M=
@@ -261,7 +243,6 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo=
 github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8=
-github.com/go-openapi/validate v0.19.12 h1:mPLM/bfbd00PGOCJlU0yJL7IulkZ+q9VjPv7U11RMQQ=
 github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4=
 github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI=
 github.com/go-openapi/validate v0.20.0 h1:pzutNCCBZGZlE+u8HD3JZyWdc/TVbtVwlWUp8/vgUKk=
@@ -272,7 +253,6 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
@@ -309,11 +289,9 @@ github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY9
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -324,9 +302,7 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@@ -346,10 +322,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -392,7 +368,6 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
@@ -440,7 +415,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -452,7 +426,6 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
@@ -470,14 +443,12 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
-github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8=
 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -488,9 +459,7 @@ github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHX
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
 github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
@@ -499,7 +468,6 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJK
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI=
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -512,10 +480,8 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg=
 github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/mitchellh/mapstructure v1.4.0 h1:7ks8ZkOP5/ujthUsT07rNv+nkLXCQWKNHuwzOAesEks=
 github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@@ -569,7 +535,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug=
 github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng=
-github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls=
 github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w=
 github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph9p/UMXK/Hk=
 github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
@@ -583,7 +548,6 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -598,8 +562,8 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.9.0 h1:Rrch9mh17XcxvEu9D9DEpb4isxjGBtcevQjKvxPRQIU=
+github.com/prometheus/client_golang v1.10.0 h1:/o0BDeWzLWXNZ+4q5gXltUvaMpJqckTa+jTNoB+z4cg=
-github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU=
+github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -613,19 +577,16 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
-github.com/prometheus/common v0.15.0 h1:4fgOnadei3EZvgRwxJ7RMpG1k1pOZth5Pc13tyspaKM=
+github.com/prometheus/common v0.18.0 h1:WCVKW7aL6LEe1uryfI9dnEc2ZqNB1Fn0ok930v0iL1Y=
-github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
+github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
-github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.3.0 h1:Uehi/mxLK0eiUc0H0++5tpMGTexB8wZ598MIgU8VpDM=
-github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
 github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
@@ -638,7 +599,6 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -647,15 +607,14 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0
 github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -668,7 +627,6 @@ github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -684,9 +642,7 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -699,11 +655,9 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go v1.2.3 h1:WbFSXLxDFKVN69Sk8t+XHGzVCD7R8UoAATR8NqZgTbk=
 github.com/ugorji/go v1.2.3/go.mod h1:5l8GZ8hZvmL4uMdy+mhCO1LjswGRYco9Q3HfuisB21A=
-github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.3 h1:/mVYEV+Jo3IZKeA5gBngN0AvNnQltEDkR+eQikkWQu0=
 github.com/ugorji/go/codec v1.2.3/go.mod h1:5FxzDJIgeiWJZslYHPj+LS1dq1ZBQVelZFnjsFGI/Uc=
@@ -723,9 +677,7 @@ go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mI
 go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
 go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
-go.mongodb.org/mongo-driver v1.3.4 h1:zs/dKNwX0gYUtzwrN9lLiR15hCO0nDwQj5xXx+vjCdE=
 go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE=
-go.mongodb.org/mongo-driver v1.4.3 h1:moga+uhicpVshTyaqY9L23E6QqwcHRUv1sqyOsoyOO8=
 go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
 go.mongodb.org/mongo-driver v1.4.4 h1:bsPHfODES+/yx2PCWzUYMH8xj6PVniPI8DQrsJuSXSs=
 go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
@@ -755,7 +707,6 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
@@ -769,7 +720,6 @@ golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+o
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -780,7 +730,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -801,19 +750,15 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw=
@@ -829,6 +774,7 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -862,30 +808,26 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4 h1:sfkvUWPNGwSV+8/fNqctR5lS2AqCSqYwXdrjCxp/dXo=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
+golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -901,7 +843,6 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -929,11 +870,9 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -948,7 +887,6 @@ google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -958,9 +896,7 @@ google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dT
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4=
 google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f h1:izedQ6yVIc5mZsRuXzmSreCOlzI0lCU1HpG8yEdMiKw=
 google.golang.org/genproto v0.0.0-20210114201628-6edceaf6022f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -968,16 +904,13 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.35.0 h1:TwIQcH3es+MojMVojxxfQ3l3OF2KzlRxML2xZq0kRo8=
@@ -988,13 +921,11 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1019,14 +950,11 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/acquisition/acquisition.go

@@ -4,7 +4,11 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
+	journalctlacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/journalctl"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/pkg/errors"
@@ -39,7 +43,17 @@ var ReaderHits = prometheus.NewCounterVec(
    source: files
    filenames:
 	- "/var/log/nginx/*.log"
+    ---
+
+	type: nginx
+	source: file
+	file:
+		filenames:
+			- /var/log/xxx
+
 	```
+
+	!!! how to handle expect mode that is not directly linked to tail/cat mode
 */
 
 /* Approach
@@ -51,66 +65,126 @@ We support acquisition in two modes :
 One DataSourceCfg can lead to multiple goroutines, hence the Tombs passing around to allow proper tracking.
 tail mode shouldn't return except on errors or when externally killed via tombs.
 cat mode will return once source has been exhausted.
-
-
- TBD in current iteration :
-  - how to deal with "file was not present at startup but might appear later" ?
 */
 
-var TAIL_MODE = "tail"
+// The interface each datasource must implement
-var CAT_MODE = "cat"
-
-type DataSourceCfg struct {
-	Mode              string            `yaml:"mode,omitempty"` //tail|cat|...
-	Filename          string            `yaml:"filename,omitempty"`
-	Filenames         []string          `yaml:"filenames,omitempty"`
-	JournalctlFilters []string          `yaml:"journalctl_filter,omitempty"`
-	Labels            map[string]string `yaml:"labels,omitempty"`
-	Profiling         bool              `yaml:"profiling,omitempty"`
-}
-
 type DataSource interface {
-	Configure(DataSourceCfg) error
+	GetMetrics() []prometheus.Collector              // Returns pointers to metrics that are managed by the module
-	/*the readers must watch the tomb (especially in tail mode) to know when to shutdown.
+	Configure([]byte, *log.Entry) error              // Configure the datasource
-	tomb is as well used to trigger general shutdown when a datasource errors */
+	ConfigureByDSN(string, string, *log.Entry) error // Configure the datasource
-	StartReading(chan types.Event, *tomb.Tomb) error
+	GetMode() string                                 // Get the mode (TAIL, CAT or SERVER)
-	Mode() string //return CAT_MODE or TAIL_MODE
+	GetName() string
-	//Not sure it makes sense to make those funcs part of the interface.
+	OneShotAcquisition(chan types.Event, *tomb.Tomb) error   // Start one shot acquisition(eg, cat a file)
-	//While 'cat' and 'tail' are the only two modes we see now, other modes might appear
+	StreamingAcquisition(chan types.Event, *tomb.Tomb) error // Start live acquisition (eg, tail a file)
-	//StartTail(chan types.Event, *tomb.Tomb) error
+	CanRun() error                                           // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense)
-	//StartCat(chan types.Event, *tomb.Tomb) error
+	Dump() interface{}
 }
 
-func DataSourceConfigure(config DataSourceCfg) (DataSource, error) {
+var AcquisitionSources = []struct {
-	if config.Mode == "" { /*default mode is tail*/
+	name  string
-		config.Mode = TAIL_MODE
+	iface func() DataSource
-	}
+}{
-
+	{
-	if len(config.Filename) > 0 || len(config.Filenames) > 0 { /*it's file acquisition*/
+		name:  "file",
-
+		iface: func() DataSource { return &fileacquisition.FileSource{} },
-		fileSrc := new(FileSource)
+	},
-		if err := fileSrc.Configure(config); err != nil {
+	{
-			return nil, errors.Wrap(err, "configuring file datasource")
+		name:  "journalctl",
-		}
+		iface: func() DataSource { return &journalctlacquisition.JournalCtlSource{} },
-		return fileSrc, nil
+	},
-	} else if len(config.JournalctlFilters) > 0 { /*it's journald acquisition*/
-
-		journaldSrc := new(JournaldSource)
-		if err := journaldSrc.Configure(config); err != nil {
-			return nil, errors.Wrap(err, "configuring journald datasource")
-		}
-		return journaldSrc, nil
-	} else {
-		return nil, fmt.Errorf("empty filename(s) and journalctl filter, malformed datasource")
-	}
 }
 
+func GetDataSourceIface(dataSourceType string) DataSource {
+	for _, source := range AcquisitionSources {
+		if source.name == dataSourceType {
+			return source.iface()
+		}
+	}
+	return nil
+}
+
+func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
+
+	//we dump it back to []byte, because we want to decode the yaml blob twice :
+	//once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
+	yamlConfig, err := yaml.Marshal(commonConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to marshal back interface")
+	}
+	if dataSrc := GetDataSourceIface(commonConfig.Source); dataSrc != nil {
+		/* this logger will then be used by the datasource at runtime */
+		clog := log.New()
+		if err := types.ConfigureLogger(clog); err != nil {
+			return nil, errors.Wrap(err, "while configuring datasource logger")
+		}
+		if commonConfig.LogLevel != nil {
+			clog.SetLevel(*commonConfig.LogLevel)
+		}
+		subLogger := clog.WithFields(log.Fields{
+			"type": commonConfig.Source,
+		})
+		/* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
+		if err := dataSrc.CanRun(); err != nil {
+			return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source)
+		}
+		/* configure the actual datasource */
+		if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
+			return nil, errors.Wrapf(err, "failed to configure datasource %s", commonConfig.Source)
+
+		}
+		return &dataSrc, nil
+	}
+	return nil, fmt.Errorf("cannot find source %s", commonConfig.Source)
+}
+
+//detectBackwardCompatAcquis : try to magically detect the type for backward compat (type was not mandatory then)
+func detectBackwardCompatAcquis(sub configuration.DataSourceCommonCfg) string {
+
+	if _, ok := sub.Config["filename"]; ok {
+		return "file"
+	}
+	if _, ok := sub.Config["filenames"]; ok {
+		return "file"
+	}
+	if _, ok := sub.Config["journalctl_filter"]; ok {
+		return "journalctl"
+	}
+	return ""
+}
+
+func LoadAcquisitionFromDSN(dsn string, label string) ([]DataSource, error) {
+	var sources []DataSource
+
+	frags := strings.Split(dsn, ":")
+	if len(frags) == 1 {
+		return nil, fmt.Errorf("%s isn't valid dsn (no protocol)", dsn)
+	}
+	dataSrc := GetDataSourceIface(frags[0])
+	if dataSrc == nil {
+		return nil, fmt.Errorf("no acquisition for protocol %s://", frags[0])
+	}
+	/* this logger will then be used by the datasource at runtime */
+	clog := log.New()
+	if err := types.ConfigureLogger(clog); err != nil {
+		return nil, errors.Wrap(err, "while configuring datasource logger")
+	}
+	subLogger := clog.WithFields(log.Fields{
+		"type": dsn,
+	})
+	err := dataSrc.ConfigureByDSN(dsn, label, subLogger)
+	if err != nil {
+		return nil, errors.Wrapf(err, "while configuration datasource for %s", dsn)
+	}
+	sources = append(sources, dataSrc)
+	return sources, nil
+}
+
+// LoadAcquisitionFromFile unmarshals the configuration item and checks its availability
 func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, error) {
 
 	var sources []DataSource
-	var acquisSources = config.AcquisitionFiles
 
-	for _, acquisFile := range acquisSources {
+	for _, acquisFile := range config.AcquisitionFiles {
 		log.Infof("loading acquisition file : %s", acquisFile)
 		yamlFile, err := os.Open(acquisFile)
 		if err != nil {
@@ -119,39 +193,61 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
 		dec := yaml.NewDecoder(yamlFile)
 		dec.SetStrict(true)
 		for {
-			sub := DataSourceCfg{}
+			var sub configuration.DataSourceCommonCfg
 			err = dec.Decode(&sub)
 			if err != nil {
 				if err == io.EOF {
 					log.Tracef("End of yaml file")
 					break
 				}
-				return nil, errors.Wrap(err, fmt.Sprintf("failed to yaml decode %s", acquisFile))
+				return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
+			}
+
+			//for backward compat ('type' was not mandatory, detect it)
+			if guessType := detectBackwardCompatAcquis(sub); guessType != "" {
+				sub.Source = guessType
+			}
+			//it's an empty item, skip it
+			if len(sub.Labels) == 0 {
+				if sub.Source == "" {
+					log.Debugf("skipping empty item in %s", acquisFile)
+					continue
+				}
+				return nil, fmt.Errorf("missing labels in %s", acquisFile)
+			}
+
+			if GetDataSourceIface(sub.Source) == nil {
+				return nil, fmt.Errorf("unknown data source %s in %s", sub.Source, acquisFile)
 			}
 			src, err := DataSourceConfigure(sub)
 			if err != nil {
-				log.Warningf("while configuring datasource : %s", err)
+				return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s", sub.Source, acquisFile)
-				continue
 			}
-			sources = append(sources, src)
+			sources = append(sources, *src)
 		}
 	}
-
 	return sources, nil
 }
 
 func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
-
 	for i := 0; i < len(sources); i++ {
 		subsrc := sources[i] //ensure its a copy
 		log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
 		AcquisTomb.Go(func() error {
 			defer types.CatchPanic("crowdsec/acquis")
-			if err := subsrc.StartReading(output, AcquisTomb); err != nil {
+			var err error
+			if subsrc.GetMode() == configuration.TAIL_MODE {
+				err = subsrc.StreamingAcquisition(output, AcquisTomb)
+			} else {
+				err = subsrc.OneShotAcquisition(output, AcquisTomb)
+			}
+			if err != nil {
 				return err
 			}
 			return nil
 		})
+		//register acquisition specific metrics
+		prometheus.MustRegister(subsrc.GetMetrics()...)
 	}
 	/*return only when acquisition is over (cat) or never (tail)*/
 	err := AcquisTomb.Wait()

pkg/acquisition/acquisition_test.go

@@ -2,137 +2,493 @@ package acquisition
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
-
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	tomb "gopkg.in/tomb.v2"
+	"gopkg.in/yaml.v2"
+	"gotest.tools/v3/assert"
 )
 
-func TestConfigLoading(t *testing.T) {
+type MockSource struct {
-	//bad filename
+	configuration.DataSourceCommonCfg `yaml:",inline"`
-	cfg := csconfig.CrowdsecServiceCfg{
+	Toto                              string `yaml:"toto"`
-		AcquisitionFiles: []string{"./tests/xxx.yaml"},
+	logger                            *log.Entry
+}
+
+func (f *MockSource) Configure(cfg []byte, logger *log.Entry) error {
+	f.logger = logger
+	if err := yaml.UnmarshalStrict(cfg, &f); err != nil {
+		return errors.Wrap(err, "while unmarshaling to reader specific config")
 	}
-	_, err := LoadAcquisitionFromFile(&cfg)
+	if f.Mode == "" {
-	assert.Contains(t, fmt.Sprintf("%s", err), "can't open ./tests/xxx.yaml: open ./tests/xxx.yaml: no such file or directory")
+		f.Mode = configuration.CAT_MODE
-	//bad config file
-	cfg = csconfig.CrowdsecServiceCfg{
-		AcquisitionFiles: []string{"./tests/test.log"},
 	}
-	_, err = LoadAcquisitionFromFile(&cfg)
+	if f.Mode != configuration.CAT_MODE && f.Mode != configuration.TAIL_MODE {
-	assert.Contains(t, fmt.Sprintf("%s", err), "failed to yaml decode ./tests/test.log: yaml: unmarshal errors")
+		return fmt.Errorf("mode %s is not supported", f.Mode)
-	//correct config file
-	cfg = csconfig.CrowdsecServiceCfg{
-		AcquisitionFiles: []string{"./tests/acquis_test.yaml"},
 	}
-	srcs, err := LoadAcquisitionFromFile(&cfg)
+	if f.Toto == "" {
-	if err != nil {
+		return fmt.Errorf("expect non-empty toto")
-		t.Fatalf("unexpected error : %s", err)
+	}
+	return nil
+}
+func (f *MockSource) GetMode() string                                         { return f.Mode }
+func (f *MockSource) OneShotAcquisition(chan types.Event, *tomb.Tomb) error   { return nil }
+func (f *MockSource) StreamingAcquisition(chan types.Event, *tomb.Tomb) error { return nil }
+func (f *MockSource) CanRun() error                                           { return nil }
+func (f *MockSource) GetMetrics() []prometheus.Collector                      { return nil }
+func (f *MockSource) Dump() interface{}                                       { return f }
+func (f *MockSource) GetName() string                                         { return "mock" }
+func (f *MockSource) ConfigureByDSN(string, string, *log.Entry) error {
+	return fmt.Errorf("not supported")
+}
+
+//copy the mocksource, but this one can't run
+type MockSourceCantRun struct {
+	MockSource
+}
+
+func (f *MockSourceCantRun) CanRun() error   { return fmt.Errorf("can't run bro") }
+func (f *MockSourceCantRun) GetName() string { return "mock_cant_run" }
+
+//appendMockSource is only used to add mock source for tests
+func appendMockSource() {
+	if GetDataSourceIface("mock") == nil {
+		mock := struct {
+			name  string
+			iface func() DataSource
+		}{
+			name:  "mock",
+			iface: func() DataSource { return &MockSource{} },
+		}
+		AcquisitionSources = append(AcquisitionSources, mock)
+	}
+	if GetDataSourceIface("mock_cant_run") == nil {
+		mock := struct {
+			name  string
+			iface func() DataSource
+		}{
+			name:  "mock_cant_run",
+			iface: func() DataSource { return &MockSourceCantRun{} },
+		}
+		AcquisitionSources = append(AcquisitionSources, mock)
 	}
-	assert.Equal(t, len(srcs), 1)
 }
 
 func TestDataSourceConfigure(t *testing.T) {
+	appendMockSource()
 	tests := []struct {
-		cfg DataSourceCfg
+		TestName      string
-		//tombState
+		RawBytes      []byte
-		config_error string
+		ExpectedError string
-		read_error   string
-		tomb_error   string
-		lines        int
 	}{
-		{ //missing filename(s)
+		{
-			cfg: DataSourceCfg{
+			TestName: "basic_valid_config",
-				Mode: CAT_MODE,
+			RawBytes: []byte(`
-			},
+mode: cat
-			config_error: "empty filename(s) and journalctl filter, malformed datasource",
+labels:
+  test: foobar
+log_level: info
+source: mock
+toto: test_value1
+`),
 		},
-		{ //missing filename(s)
+		{
-			cfg: DataSourceCfg{
+			TestName: "basic_debug_config",
-				Mode: TAIL_MODE,
+			RawBytes: []byte(`
-			},
+mode: cat
-			config_error: "empty filename(s) and journalctl filter, malformed datasource",
+labels:
+  test: foobar
+log_level: debug
+source: mock
+toto: test_value1
+`),
 		},
-		{ //bad mode(s)
+		{
-			cfg: DataSourceCfg{
+			TestName: "basic_tailmode_config",
-				Filename: "./tests/test.log",
+			RawBytes: []byte(`
-				Mode:     "ratata",
+mode: tail
-			},
+labels:
-			config_error: "configuring file datasource: unknown mode ratata for file acquisition",
+  test: foobar
+log_level: debug
+source: mock
+toto: test_value1
+`),
 		},
-		{ //ok test
+		{
-			cfg: DataSourceCfg{
+			TestName: "bad_mode_config",
-				Mode:     CAT_MODE,
+			RawBytes: []byte(`
-				Filename: "./tests/test.log",
+mode: ratata
-			},
+labels:
+  test: foobar
+log_level: debug
+source: mock
+toto: test_value1
+`),
+			ExpectedError: "failed to configure datasource mock: mode ratata is not supported",
 		},
-		{ //missing mode, default to CAT_MODE
+		{
-			cfg: DataSourceCfg{
+			TestName: "bad_type_config",
-				Filename: "./tests/test.log",
+			RawBytes: []byte(`
-			},
+mode: cat
+labels:
+  test: foobar
+log_level: debug
+source: tutu
+`),
+			ExpectedError: "cannot find source tutu",
 		},
-		{ //ok test for journalctl
+		{
-			cfg: DataSourceCfg{
+			TestName: "mismatch_config",
-				Mode:              CAT_MODE,
+			RawBytes: []byte(`
-				JournalctlFilters: []string{"-test.run=TestSimJournalctlCatOneLine", "--"},
+mode: cat
-			},
+labels:
+  test: foobar
+log_level: debug
+source: mock
+wowo: ajsajasjas
+`),
+			ExpectedError: "field wowo not found in type acquisition.MockSource",
+		},
+		{
+			TestName: "cant_run_error",
+			RawBytes: []byte(`
+mode: cat
+labels:
+  test: foobar
+log_level: debug
+source: mock_cant_run
+wowo: ajsajasjas
+`),
+			ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
 		},
 	}
 
-	for tidx, test := range tests {
+	for _, test := range tests {
-
+		common := configuration.DataSourceCommonCfg{}
-		srcs, err := DataSourceConfigure(test.cfg)
+		yaml.Unmarshal(test.RawBytes, &common)
-		if test.config_error != "" {
+		ds, err := DataSourceConfigure(common)
-			assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
+		if test.ExpectedError != "" {
-			log.Infof("expected config error ok : %s", test.config_error)
+			if err == nil {
-			continue
+				t.Fatalf("expected error %s, got none", test.ExpectedError)
+			}
+			if !strings.Contains(err.Error(), test.ExpectedError) {
+				t.Fatalf("%s : expected error '%s' in '%s'", test.TestName, test.ExpectedError, err.Error())
+			} else {
+				continue
+			}
 		} else {
 			if err != nil {
-				t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
+				t.Fatalf("%s : unexpected error '%s'", test.TestName, err)
 			}
 		}
 
-		//check we got the expected mode
+		switch test.TestName {
-		if tests[tidx].cfg.Mode == "" {
+		case "basic_valid_config":
-			tests[tidx].cfg.Mode = TAIL_MODE
+			mock := (*ds).Dump().(*MockSource)
+			assert.Equal(t, mock.Toto, "test_value1")
+			assert.Equal(t, mock.Mode, "cat")
+			assert.Equal(t, mock.logger.Logger.Level, log.InfoLevel)
+			assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"})
+		case "basic_debug_config":
+			mock := (*ds).Dump().(*MockSource)
+			assert.Equal(t, mock.Toto, "test_value1")
+			assert.Equal(t, mock.Mode, "cat")
+			assert.Equal(t, mock.logger.Logger.Level, log.DebugLevel)
+			assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"})
+		case "basic_tailmode_config":
+			mock := (*ds).Dump().(*MockSource)
+			assert.Equal(t, mock.Toto, "test_value1")
+			assert.Equal(t, mock.Mode, "tail")
+			assert.Equal(t, mock.logger.Logger.Level, log.DebugLevel)
+			assert.DeepEqual(t, mock.Labels, map[string]string{"test": "foobar"})
 		}
-		assert.Equal(t, srcs.Mode(), tests[tidx].cfg.Mode)
+	}
+}
 
-		out := make(chan types.Event)
+func TestLoadAcquisitionFromFile(t *testing.T) {
-		tomb := tomb.Tomb{}
+	appendMockSource()
-
+	tests := []struct {
-		go func() {
+		TestName      string
-			err = StartAcquisition([]DataSource{srcs}, out, &tomb)
+		Config        csconfig.CrowdsecServiceCfg
-			if test.read_error != "" {
+		ExpectedError string
-				assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
+		ExpectedLen   int
-				log.Infof("expected read error ok : %s", test.read_error)
+	}{
+		{
+			TestName: "non_existent_file",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"does_not_exist"},
+			},
+			ExpectedError: "can't open does_not_exist",
+			ExpectedLen:   0,
+		},
+		{
+			TestName: "invalid_yaml_file",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"test_files/badyaml.yaml"},
+			},
+			ExpectedError: "failed to yaml decode test_files/badyaml.yaml: yaml: unmarshal errors",
+			ExpectedLen:   0,
+		},
+		{
+			TestName: "invalid_empty_yaml",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"test_files/emptyitem.yaml"},
+			},
+			ExpectedLen: 0,
+		},
+		{
+			TestName: "basic_valid",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"test_files/basic_filemode.yaml"},
+			},
+			ExpectedLen: 2,
+		},
+		{
+			TestName: "missing_labels",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"test_files/missing_labels.yaml"},
+			},
+			ExpectedError: "missing labels in test_files/missing_labels.yaml",
+		},
+		{
+			TestName: "backward_compat",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"test_files/backward_compat.yaml"},
+			},
+			ExpectedLen: 2,
+		},
+		{
+			TestName: "bad_type",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"test_files/bad_source.yaml"},
+			},
+			ExpectedError: "unknown data source does_not_exist in test_files/bad_source.yaml",
+		},
+		{
+			TestName: "invalid_filetype_config",
+			Config: csconfig.CrowdsecServiceCfg{
+				AcquisitionFiles: []string{"test_files/bad_filetype.yaml"},
+			},
+			ExpectedError: "while configuring datasource of type file from test_files/bad_filetype.yaml",
+		},
+	}
+	for _, test := range tests {
+		dss, err := LoadAcquisitionFromFile(&test.Config)
+		if test.ExpectedError != "" {
+			if err == nil {
+				t.Fatalf("expected error %s, got none", test.ExpectedError)
+			}
+			if !strings.Contains(err.Error(), test.ExpectedError) {
+				t.Fatalf("%s : expected error '%s' in '%s'", test.TestName, test.ExpectedError, err.Error())
 			} else {
-				if err != nil {
+				continue
-					log.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
-				}
 			}
-		}()
-
-		log.Printf("kill iiittt")
-		//we're actually not interested in the result :)
-		tomb.Kill(nil)
-		time.Sleep(1 * time.Second)
-
-		if test.tomb_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
-			log.Infof("expected tomb error ok : %s", test.read_error)
-			continue
 		} else {
-			if tomb.Err() != nil {
+			if err != nil {
-				t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
+				t.Fatalf("%s : unexpected error '%s'", test.TestName, err)
 			}
 		}
+		if len(dss) != test.ExpectedLen {
+			t.Fatalf("%s : expected %d datasources got %d", test.TestName, test.ExpectedLen, len(dss))
+		}
 
 	}
-
+}
+
+/*
+ test start acquisition :
+  - create mock parser in cat mode : start acquisition, check it returns, count items in chan
+  - create mock parser in tail mode : start acquisition, sleep, check item count, tomb kill it, wait for it to return
+*/
+
+type MockCat struct {
+	configuration.DataSourceCommonCfg `yaml:",inline"`
+	logger                            *log.Entry
+}
+
+func (f *MockCat) Configure(cfg []byte, logger *log.Entry) error {
+	f.logger = logger
+	if f.Mode == "" {
+		f.Mode = configuration.CAT_MODE
+	}
+	if f.Mode != configuration.CAT_MODE {
+		return fmt.Errorf("mode %s is not supported", f.Mode)
+	}
+	return nil
+}
+func (f *MockCat) GetName() string { return "mock_cat" }
+func (f *MockCat) GetMode() string { return "cat" }
+func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
+	for i := 0; i < 10; i++ {
+		evt := types.Event{}
+		evt.Line.Src = "test"
+		out <- evt
+	}
+	return nil
+}
+func (f *MockCat) StreamingAcquisition(chan types.Event, *tomb.Tomb) error {
+	return fmt.Errorf("can't run in tail")
+}
+func (f *MockCat) CanRun() error                                   { return nil }
+func (f *MockCat) GetMetrics() []prometheus.Collector              { return nil }
+func (f *MockCat) Dump() interface{}                               { return f }
+func (f *MockCat) ConfigureByDSN(string, string, *log.Entry) error { return fmt.Errorf("not supported") }
+
+//----
+
+type MockTail struct {
+	configuration.DataSourceCommonCfg `yaml:",inline"`
+	logger                            *log.Entry
+}
+
+func (f *MockTail) Configure(cfg []byte, logger *log.Entry) error {
+	f.logger = logger
+	if f.Mode == "" {
+		f.Mode = configuration.TAIL_MODE
+	}
+	if f.Mode != configuration.TAIL_MODE {
+		return fmt.Errorf("mode %s is not supported", f.Mode)
+	}
+	return nil
+}
+func (f *MockTail) GetName() string { return "mock_tail" }
+func (f *MockTail) GetMode() string { return "tail" }
+func (f *MockTail) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
+	return fmt.Errorf("can't run in cat mode")
+}
+func (f *MockTail) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	for i := 0; i < 10; i++ {
+		evt := types.Event{}
+		evt.Line.Src = "test"
+		out <- evt
+	}
+	select {
+	case <-t.Dying():
+		return nil
+	}
+}
+func (f *MockTail) CanRun() error                      { return nil }
+func (f *MockTail) GetMetrics() []prometheus.Collector { return nil }
+func (f *MockTail) Dump() interface{}                  { return f }
+func (f *MockTail) ConfigureByDSN(string, string, *log.Entry) error {
+	return fmt.Errorf("not supported")
+}
+
+//func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error {
+
+func TestStartAcquisitionCat(t *testing.T) {
+	sources := []DataSource{
+		&MockCat{},
+	}
+	out := make(chan types.Event)
+	acquisTomb := tomb.Tomb{}
+
+	go func() {
+		if err := StartAcquisition(sources, out, &acquisTomb); err != nil {
+			t.Fatalf("unexpected error")
+		}
+	}()
+
+	count := 0
+READLOOP:
+	for {
+		select {
+		case <-out:
+			count++
+		case <-time.After(1 * time.Second):
+			break READLOOP
+		}
+	}
+	if count != 10 {
+		t.Fatalf("expected 10 results, got %d", count)
+	}
+}
+
+func TestStartAcquisitionTail(t *testing.T) {
+	sources := []DataSource{
+		&MockTail{},
+	}
+	out := make(chan types.Event)
+	acquisTomb := tomb.Tomb{}
+
+	go func() {
+		if err := StartAcquisition(sources, out, &acquisTomb); err != nil {
+			t.Fatalf("unexpected error")
+		}
+	}()
+
+	count := 0
+READLOOP:
+	for {
+		select {
+		case <-out:
+			count++
+		case <-time.After(1 * time.Second):
+			break READLOOP
+		}
+	}
+	if count != 10 {
+		t.Fatalf("expected 10 results, got %d", count)
+	}
+	acquisTomb.Kill(nil)
+	time.Sleep(1 * time.Second)
+	if acquisTomb.Err() != nil {
+		t.Fatalf("unexpected tomb error %s (should be dead)", acquisTomb.Err())
+	}
+}
+
+//
+type MockTailError struct {
+	MockTail
+}
+
+func (f *MockTailError) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	for i := 0; i < 10; i++ {
+		evt := types.Event{}
+		evt.Line.Src = "test"
+		out <- evt
+	}
+	t.Kill(fmt.Errorf("got error (tomb)"))
+	return fmt.Errorf("got error")
+}
+
+func TestStartAcquisitionTailError(t *testing.T) {
+	sources := []DataSource{
+		&MockTailError{},
+	}
+	out := make(chan types.Event)
+	acquisTomb := tomb.Tomb{}
+
+	go func() {
+		if err := StartAcquisition(sources, out, &acquisTomb); err != nil && err.Error() != "got error (tomb)" {
+			t.Fatalf("expected error, got '%s'", err.Error())
+		}
+	}()
+
+	count := 0
+READLOOP:
+	for {
+		select {
+		case <-out:
+			count++
+		case <-time.After(1 * time.Second):
+			break READLOOP
+		}
+	}
+	if count != 10 {
+		t.Fatalf("expected 10 results, got %d", count)
+	}
+	//acquisTomb.Kill(nil)
+	time.Sleep(1 * time.Second)
+	if acquisTomb.Err().Error() != "got error (tomb)" {
+		t.Fatalf("didn't got expected error, got '%s'", acquisTomb.Err().Error())
+	}
 }

pkg/acquisition/configuration/configuration.go

@@ -0,0 +1,17 @@
+package configuration
+
+import (
+	log "github.com/sirupsen/logrus"
+)
+
+type DataSourceCommonCfg struct {
+	Mode     string                 `yaml:"mode,omitempty"`
+	Labels   map[string]string      `yaml:"labels,omitempty"`
+	LogLevel *log.Level             `yaml:"log_level,omitempty"`
+	Source   string                 `yaml:"source,omitempty"`
+	Config   map[string]interface{} `yaml:",inline"` //to keep the datasource-specific configuration directives
+}
+
+var TAIL_MODE = "tail"
+var CAT_MODE = "cat"
+var SERVER_MODE = "server" // No difference with tail, just a bit more verbose

pkg/acquisition/file_reader.go

@@ -1,227 +0,0 @@
-package acquisition
-
-import (
-	"bufio"
-	"compress/gzip"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/pkg/errors"
-	"golang.org/x/sys/unix"
-
-	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
-
-	"github.com/crowdsecurity/crowdsec/pkg/types"
-	"github.com/nxadm/tail"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/prometheus/client_golang/prometheus"
-	tomb "gopkg.in/tomb.v2"
-)
-
-type FileSource struct {
-	Config DataSourceCfg
-	tails  []*tail.Tail
-	Files  []string
-}
-
-func (f *FileSource) Configure(Config DataSourceCfg) error {
-	f.Config = Config
-	if len(Config.Filename) == 0 && len(Config.Filenames) == 0 {
-		return fmt.Errorf("no filename or filenames")
-	}
-
-	//let's deal with the array no matter what
-	if len(Config.Filename) != 0 {
-		Config.Filenames = append(Config.Filenames, Config.Filename)
-	}
-
-	for _, fexpr := range Config.Filenames {
-		files, err := filepath.Glob(fexpr)
-		if err != nil {
-			return errors.Wrapf(err, "while globbing %s", fexpr)
-		}
-		if len(files) == 0 {
-			log.Warningf("[file datasource] no results for %s", fexpr)
-			continue
-		}
-
-		for _, file := range files {
-			/*check that we can read said file*/
-			if err := unix.Access(file, unix.R_OK); err != nil {
-				return fmt.Errorf("unable to open %s : %s", file, err)
-			}
-			log.Infof("[file datasource] opening file '%s'", file)
-
-			if f.Config.Mode == TAIL_MODE {
-				tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: 2}})
-				if err != nil {
-					log.Errorf("[file datasource] skipping %s : %v", file, err)
-					continue
-				}
-				f.Files = append(f.Files, file)
-				f.tails = append(f.tails, tail)
-			} else if f.Config.Mode == CAT_MODE {
-				//simply check that the file exists, it will be read differently
-				if _, err := os.Stat(file); err != nil {
-					return fmt.Errorf("can't open file %s : %s", file, err)
-				}
-				f.Files = append(f.Files, file)
-			} else {
-				return fmt.Errorf("unknown mode %s for file acquisition", f.Config.Mode)
-			}
-
-		}
-	}
-	if len(f.Files) == 0 {
-		return fmt.Errorf("no files to read for %+v", Config.Filenames)
-	}
-
-	return nil
-}
-
-func (f *FileSource) Mode() string {
-	return f.Config.Mode
-}
-
-func (f *FileSource) StartReading(out chan types.Event, t *tomb.Tomb) error {
-
-	if f.Config.Mode == CAT_MODE {
-		return f.StartCat(out, t)
-	} else if f.Config.Mode == TAIL_MODE {
-		return f.StartTail(out, t)
-	} else {
-		return fmt.Errorf("unknown mode '%s' for file acquisition", f.Config.Mode)
-	}
-}
-
-/*A tail-mode file reader (tail) */
-func (f *FileSource) StartTail(output chan types.Event, AcquisTomb *tomb.Tomb) error {
-	log.Debugf("starting file tail with %d items", len(f.tails))
-	for i := 0; i < len(f.tails); i++ {
-		idx := i
-		log.Debugf("starting %d", idx)
-		AcquisTomb.Go(func() error {
-			defer types.CatchPanic("crowdsec/acquis/tailfile")
-			return f.TailOneFile(output, AcquisTomb, idx)
-		})
-	}
-	return nil
-}
-
-/*A one shot file reader (cat) */
-func (f *FileSource) StartCat(output chan types.Event, AcquisTomb *tomb.Tomb) error {
-	for i := 0; i < len(f.Files); i++ {
-		idx := i
-		log.Debugf("starting %d", idx)
-		AcquisTomb.Go(func() error {
-			defer types.CatchPanic("crowdsec/acquis/catfile")
-			return f.CatOneFile(output, AcquisTomb, idx)
-		})
-	}
-	return nil
-}
-
-/*A tail-mode file reader (tail) */
-func (f *FileSource) TailOneFile(output chan types.Event, AcquisTomb *tomb.Tomb, idx int) error {
-
-	file := f.Files[idx]
-	tail := f.tails[idx]
-
-	clog := log.WithFields(log.Fields{
-		"acquisition file": f.Files[idx],
-	})
-	clog.Debugf("starting")
-
-	timeout := time.Tick(1 * time.Second)
-
-	for {
-		l := types.Line{}
-		select {
-		case <-AcquisTomb.Dying(): //we are being killed by main
-			clog.Infof("file datasource %s stopping", file)
-			if err := tail.Stop(); err != nil {
-				clog.Errorf("error in stop : %s", err)
-			}
-			return nil
-		case <-tail.Tomb.Dying(): //our tailer is dying
-			clog.Warningf("File reader of %s died", file)
-			AcquisTomb.Kill(fmt.Errorf("dead reader for %s", file))
-			return fmt.Errorf("reader for %s is dead", file)
-		case line := <-tail.Lines:
-			if line == nil {
-				clog.Debugf("Nil line")
-				return fmt.Errorf("tail for %s is empty", file)
-			}
-			if line.Err != nil {
-				log.Warningf("fetch error : %v", line.Err)
-				return line.Err
-			}
-			if line.Text == "" { //skip empty lines
-				continue
-			}
-			ReaderHits.With(prometheus.Labels{"source": file}).Inc()
-
-			l.Raw = line.Text
-			l.Labels = f.Config.Labels
-			l.Time = line.Time
-			l.Src = file
-			l.Process = true
-			//we're tailing, it must be real time logs
-			log.Debugf("pushing %+v", l)
-			output <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
-		case <-timeout:
-			//time out, shall we do stuff ?
-			clog.Debugf("timeout")
-		}
-	}
-}
-
-/*A one shot file reader (cat) */
-func (f *FileSource) CatOneFile(output chan types.Event, AcquisTomb *tomb.Tomb, idx int) error {
-	var scanner *bufio.Scanner
-
-	log.Infof("reading %s at once", f.Files[idx])
-	file := f.Files[idx]
-
-	clog := log.WithFields(log.Fields{
-		"file": file,
-	})
-	fd, err := os.Open(file)
-	defer fd.Close()
-	if err != nil {
-		clog.Errorf("Failed opening file: %s", err)
-		return errors.Wrapf(err, "failed opening %s", f.Files[idx])
-	}
-
-	if strings.HasSuffix(file, ".gz") {
-		gz, err := gzip.NewReader(fd)
-		if err != nil {
-			clog.Errorf("Failed to read gz file: %s", err)
-			return errors.Wrapf(err, "failed to read gz %s", f.Files[idx])
-		}
-		defer gz.Close()
-		scanner = bufio.NewScanner(gz)
-
-	} else {
-		scanner = bufio.NewScanner(fd)
-	}
-	scanner.Split(bufio.ScanLines)
-	for scanner.Scan() {
-		log.Tracef("line %s", scanner.Text())
-		l := types.Line{}
-		l.Raw = scanner.Text()
-		l.Time = time.Now()
-		l.Src = file
-		l.Labels = f.Config.Labels
-		l.Process = true
-		ReaderHits.With(prometheus.Labels{"source": file}).Inc()
-		//we're reading logs at once, it must be time-machine buckets
-		output <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.TIMEMACHINE}
-	}
-	AcquisTomb.Kill(nil)
-	return nil
-}

pkg/acquisition/file_reader_test.go

@@ -1,383 +0,0 @@
-package acquisition
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/crowdsecurity/crowdsec/pkg/types"
-	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
-	tomb "gopkg.in/tomb.v2"
-)
-
-func TestAcquisCat(t *testing.T) {
-
-	tests := []struct {
-		cfg DataSourceCfg
-		//tombState
-		config_error string
-		read_error   string
-		tomb_error   string
-		lines        int
-	}{
-		{ //missing filename(s)
-			cfg: DataSourceCfg{
-				Mode: CAT_MODE,
-			},
-			config_error: "no filename or filenames",
-		},
-		{ //forbiden file
-			cfg: DataSourceCfg{
-				Mode:     CAT_MODE,
-				Filename: "/etc/shadow",
-			},
-			config_error: "unable to open /etc/shadow : permission denied",
-		},
-		{ //bad regexp
-			cfg: DataSourceCfg{
-				Filename: "[a-",
-				Mode:     CAT_MODE,
-			},
-			config_error: "while globbing [a-: syntax error in pattern",
-		},
-		{ //inexisting file
-			cfg: DataSourceCfg{
-				Filename: "/does/not/exists",
-				Mode:     CAT_MODE,
-			},
-			config_error: "no files to read for [/does/not/exists]",
-		},
-		{ //ok file
-			cfg: DataSourceCfg{
-				Filename: "./tests/test.log",
-				Mode:     CAT_MODE,
-			},
-			lines: 1,
-		},
-		{ //invalid gz
-			cfg: DataSourceCfg{
-				Filename: "./tests/badlog.gz",
-				Mode:     CAT_MODE,
-			},
-			lines:      0,
-			tomb_error: "failed to read gz ./tests/badlog.gz: EOF",
-		},
-		{ //good gz
-			cfg: DataSourceCfg{
-				Filename: "./tests/test.log.gz",
-				Mode:     CAT_MODE,
-			},
-			lines: 1,
-		},
-	}
-
-	for tidx, test := range tests {
-		fileSrc := new(FileSource)
-		err := fileSrc.Configure(test.cfg)
-		if test.config_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
-			log.Infof("expected config error ok : %s", test.config_error)
-			continue
-		} else {
-			if err != nil {
-				t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
-			}
-		}
-
-		out := make(chan types.Event)
-		tomb := tomb.Tomb{}
-		count := 0
-
-		err = fileSrc.StartReading(out, &tomb)
-		if test.read_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
-			log.Infof("expected read error ok : %s", test.read_error)
-			continue
-		} else {
-			if err != nil {
-				t.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
-			}
-		}
-
-	READLOOP:
-		for {
-			select {
-			case <-out:
-				count++
-			case <-time.After(1 * time.Second):
-				break READLOOP
-			}
-		}
-
-		if count != test.lines {
-			t.Fatalf("%d/%d expected %d line read, got %d", tidx, len(tests), test.lines, count)
-		}
-
-		if test.tomb_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
-			log.Infof("expected tomb error ok : %s", test.read_error)
-			continue
-		} else {
-			if tomb.Err() != nil {
-				t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
-			}
-		}
-
-	}
-
-}
-
-func TestTailKill(t *testing.T) {
-	cfg := DataSourceCfg{
-		Filename: "./tests/test.log",
-		Mode:     TAIL_MODE,
-	}
-
-	fileSrc := new(FileSource)
-	err := fileSrc.Configure(cfg)
-	if err != nil {
-		t.Fatalf("unexpected config error %s", err)
-	}
-
-	out := make(chan types.Event)
-	tb := tomb.Tomb{}
-
-	err = fileSrc.StartReading(out, &tb)
-	if err != nil {
-		t.Fatalf("unexpected read error %s", err)
-	}
-	time.Sleep(1 * time.Second)
-	if tb.Err() != tomb.ErrStillAlive {
-		t.Fatalf("unexpected tomb error %s (should be alive)", tb.Err())
-	}
-	//kill it :>
-	tb.Kill(nil)
-	time.Sleep(1 * time.Second)
-	if tb.Err() != nil {
-		t.Fatalf("unexpected tomb error %s (should be dead)", tb.Err())
-	}
-
-}
-
-func TestTailKillBis(t *testing.T) {
-	cfg := DataSourceCfg{
-		Filename: "./tests/test.log",
-		Mode:     TAIL_MODE,
-	}
-
-	fileSrc := new(FileSource)
-	err := fileSrc.Configure(cfg)
-	if err != nil {
-		t.Fatalf("unexpected config error %s", err)
-	}
-
-	out := make(chan types.Event)
-	tb := tomb.Tomb{}
-
-	err = fileSrc.StartReading(out, &tb)
-	if err != nil {
-		t.Fatalf("unexpected read error %s", err)
-	}
-	time.Sleep(1 * time.Second)
-	if tb.Err() != tomb.ErrStillAlive {
-		t.Fatalf("unexpected tomb error %s (should be alive)", tb.Err())
-	}
-	//kill the underlying tomb of tailer
-	fileSrc.tails[0].Kill(fmt.Errorf("ratata"))
-	time.Sleep(1 * time.Second)
-	//it can be two errors :
-	if !strings.Contains(fmt.Sprintf("%s", tb.Err()), "dead reader for ./tests/test.log") &&
-		!strings.Contains(fmt.Sprintf("%s", tb.Err()), "tail for ./tests/test.log is empty") {
-		t.Fatalf("unexpected error : %s", tb.Err())
-	}
-
-}
-
-func TestTailRuntime(t *testing.T) {
-	//log.SetLevel(log.TraceLevel)
-
-	cfg := DataSourceCfg{
-		Filename: "./tests/test.log",
-		Mode:     TAIL_MODE,
-	}
-
-	fileSrc := new(FileSource)
-	err := fileSrc.Configure(cfg)
-	if err != nil {
-		t.Fatalf("unexpected config error %s", err)
-	}
-
-	out := make(chan types.Event)
-	tb := tomb.Tomb{}
-	count := 0
-
-	err = fileSrc.StartReading(out, &tb)
-	if err != nil {
-		t.Fatalf("unexpected read error %s", err)
-	}
-
-	time.Sleep(1 * time.Second)
-	//write data
-	f, err := os.OpenFile(cfg.Filename, os.O_APPEND|os.O_WRONLY, 0644)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for i := 0; i < 5; i++ {
-		_, err := f.WriteString(fmt.Sprintf("ratata%d\n", i))
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-	f.Close()
-
-READLOOP:
-	for {
-		select {
-		case <-out:
-			count++
-		case <-time.After(1 * time.Second):
-			break READLOOP
-		}
-	}
-
-	if count != 5 {
-		t.Fatalf("expected %d line read, got %d", 5, count)
-	}
-
-	if tb.Err() != tomb.ErrStillAlive {
-		t.Fatalf("unexpected tomb error %s", tb.Err())
-	}
-
-	/*reset the file*/
-	f, err = os.OpenFile(cfg.Filename, os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, err = f.WriteString("one log line\n")
-	if err != nil {
-		t.Fatal(err)
-	}
-	f.Close()
-}
-
-func TestAcquisTail(t *testing.T) {
-
-	tests := []struct {
-		cfg DataSourceCfg
-		//tombState
-		config_error string
-		read_error   string
-		tomb_error   string
-		lines        int
-	}{
-		{ //missing filename(s)
-			cfg: DataSourceCfg{
-				Mode: TAIL_MODE,
-			},
-			config_error: "no filename or filenames",
-		},
-		{ //forbiden file
-			cfg: DataSourceCfg{
-				Mode:     TAIL_MODE,
-				Filename: "/etc/shadow",
-			},
-			config_error: "unable to open /etc/shadow : permission denied",
-		},
-		{ //bad regexp
-			cfg: DataSourceCfg{
-				Filename: "[a-",
-				Mode:     TAIL_MODE,
-			},
-			config_error: "while globbing [a-: syntax error in pattern",
-		},
-		{ //inexisting file
-			cfg: DataSourceCfg{
-				Filename: "/does/not/exists",
-				Mode:     TAIL_MODE,
-			},
-			config_error: "no files to read for [/does/not/exists]",
-		},
-		{ //ok file
-			cfg: DataSourceCfg{
-				Filename: "./tests/test.log",
-				Mode:     TAIL_MODE,
-			},
-			lines:      0,
-			tomb_error: "still alive",
-		},
-		{ //invalid gz
-			cfg: DataSourceCfg{
-				Filename: "./tests/badlog.gz",
-				Mode:     TAIL_MODE,
-			},
-			lines:      0,
-			tomb_error: "still alive",
-		},
-		{ //good gz
-			cfg: DataSourceCfg{
-				Filename: "./tests/test.log.gz",
-				Mode:     TAIL_MODE,
-			},
-			lines:      0,
-			tomb_error: "still alive",
-		},
-	}
-
-	for tidx, test := range tests {
-		fileSrc := new(FileSource)
-		err := fileSrc.Configure(test.cfg)
-		if test.config_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
-			log.Infof("expected config error ok : %s", test.config_error)
-			continue
-		} else {
-			if err != nil {
-				t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
-			}
-		}
-
-		out := make(chan types.Event)
-		tomb := tomb.Tomb{}
-		count := 0
-
-		err = fileSrc.StartReading(out, &tomb)
-		if test.read_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
-			log.Infof("expected read error ok : %s", test.read_error)
-			continue
-		} else {
-			if err != nil {
-				t.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
-			}
-		}
-
-	READLOOP:
-		for {
-			select {
-			case <-out:
-				count++
-			case <-time.After(1 * time.Second):
-				break READLOOP
-			}
-		}
-
-		if count != test.lines {
-			t.Fatalf("%d/%d expected %d line read, got %d", tidx, len(tests), test.lines, count)
-		}
-
-		if test.tomb_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
-			log.Infof("expected tomb error ok : %s", test.read_error)
-			continue
-		} else {
-			if tomb.Err() != nil {
-				t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
-			}
-		}
-
-	}
-
-}

pkg/acquisition/journalctl_reader.go

@@ -1,174 +0,0 @@
-package acquisition
-
-import (
-	"bufio"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os/exec"
-	"strings"
-	"time"
-
-	log "github.com/sirupsen/logrus"
-
-	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	tomb "gopkg.in/tomb.v2"
-)
-
-/*
- journald/systemd support :
-
- systemd has its own logging system, which stores files in non-text mode.
- To be able to read those, we're going to read the output of journalctl, see https://github.com/crowdsecurity/crowdsec/issues/423
-
-
- TBD :
-  - handle journalctl errors
-*/
-
-type JournaldSource struct {
-	Config  DataSourceCfg
-	Cmd     *exec.Cmd
-	Stdout  io.ReadCloser
-	Stderr  io.ReadCloser
-	Decoder *json.Decoder
-	SrcName string
-}
-
-var JOURNALD_CMD = "journalctl"
-var JOURNALD_DEFAULT_TAIL_ARGS = []string{"--follow"}
-var JOURNALD_DEFAULT_CAT_ARGS = []string{}
-
-func (j *JournaldSource) Configure(config DataSourceCfg) error {
-	var journalArgs []string
-
-	j.Config = config
-	if config.JournalctlFilters == nil {
-		return fmt.Errorf("journalctl_filter shouldn't be empty")
-	}
-
-	if j.Config.Mode == TAIL_MODE {
-		journalArgs = JOURNALD_DEFAULT_TAIL_ARGS
-	} else if j.Config.Mode == CAT_MODE {
-		journalArgs = JOURNALD_DEFAULT_CAT_ARGS
-	} else {
-		return fmt.Errorf("unknown mode '%s' for journald source", j.Config.Mode)
-	}
-	journalArgs = append(journalArgs, config.JournalctlFilters...)
-
-	j.Cmd = exec.Command(JOURNALD_CMD, journalArgs...)
-	j.Stderr, _ = j.Cmd.StderrPipe()
-	j.Stdout, _ = j.Cmd.StdoutPipe()
-	j.SrcName = fmt.Sprintf("journalctl-%s", strings.Join(config.JournalctlFilters, "."))
-	log.Infof("[journald datasource] Configured with filters : %+v", journalArgs)
-	log.Debugf("cmd path : %s", j.Cmd.Path)
-	log.Debugf("cmd args : %+v", j.Cmd.Args)
-
-	return nil
-}
-
-func (j *JournaldSource) Mode() string {
-	return j.Config.Mode
-}
-
-func (j *JournaldSource) readOutput(out chan types.Event, t *tomb.Tomb) error {
-
-	/*
-		todo : handle the channel
-	*/
-	clog := log.WithFields(log.Fields{
-		"acquisition file": j.SrcName,
-	})
-	if err := j.Cmd.Start(); err != nil {
-		clog.Errorf("failed to start journalctl: %s", err)
-		return errors.Wrapf(err, "starting journalctl (%s)", j.SrcName)
-	}
-
-	readErr := make(chan error)
-
-	/*read stderr*/
-	go func() {
-		scanner := bufio.NewScanner(j.Stderr)
-		if scanner == nil {
-			readErr <- fmt.Errorf("failed to create stderr scanner")
-			return
-		}
-		for scanner.Scan() {
-			txt := scanner.Text()
-			clog.Warningf("got stderr message : %s", txt)
-			readErr <- fmt.Errorf(txt)
-		}
-	}()
-	/*read stdout*/
-	go func() {
-		scanner := bufio.NewScanner(j.Stdout)
-		if scanner == nil {
-			readErr <- fmt.Errorf("failed to create stdout scanner")
-			return
-		}
-		for scanner.Scan() {
-			l := types.Line{}
-			ReaderHits.With(prometheus.Labels{"source": j.SrcName}).Inc()
-			l.Raw = scanner.Text()
-			clog.Debugf("getting one line : %s", l.Raw)
-			l.Labels = j.Config.Labels
-			l.Time = time.Now()
-			l.Src = j.SrcName
-			l.Process = true
-			evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
-			out <- evt
-		}
-		clog.Debugf("finished reading from journalctl")
-		if err := scanner.Err(); err != nil {
-			clog.Debugf("got an error while reading %s : %s", j.SrcName, err)
-			readErr <- err
-			return
-		}
-		readErr <- nil
-	}()
-
-	for {
-		select {
-		case <-t.Dying():
-			clog.Debugf("journalctl datasource %s stopping", j.SrcName)
-			return nil
-		case err := <-readErr:
-			clog.Debugf("the subroutine returned, leave as well")
-			if err != nil {
-				clog.Warningf("journalctl reader error : %s", err)
-				t.Kill(err)
-			}
-			return err
-		}
-	}
-}
-
-func (j *JournaldSource) StartReading(out chan types.Event, t *tomb.Tomb) error {
-
-	if j.Config.Mode == CAT_MODE {
-		return j.StartCat(out, t)
-	} else if j.Config.Mode == TAIL_MODE {
-		return j.StartTail(out, t)
-	} else {
-		return fmt.Errorf("unknown mode '%s' for file acquisition", j.Config.Mode)
-	}
-}
-
-func (j *JournaldSource) StartCat(out chan types.Event, t *tomb.Tomb) error {
-	t.Go(func() error {
-		defer types.CatchPanic("crowdsec/acquis/tailjournalctl")
-		return j.readOutput(out, t)
-	})
-	return nil
-}
-
-func (j *JournaldSource) StartTail(out chan types.Event, t *tomb.Tomb) error {
-	t.Go(func() error {
-		defer types.CatchPanic("crowdsec/acquis/catjournalctl")
-		return j.readOutput(out, t)
-	})
-	return nil
-}

pkg/acquisition/journalctl_reader_test.go

@@ -1,238 +0,0 @@
-package acquisition
-
-import (
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/crowdsecurity/crowdsec/pkg/types"
-	log "github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
-	tomb "gopkg.in/tomb.v2"
-)
-
-/*
- As we can't decently run journalctl in the CI but we still need to test the command execution aspect :
-  - we create tests 'output only' (cf. TestSimJournalctlCat) that just produce outputs
-  - we run ourselves (os.Args[0]) with specific args to call specific 'output only' tests
-  - and this is how we test the behavior
-*/
-
-//14 lines of sshd logs
-var testjournalctl_output_1 string = `-- Logs begin at Fri 2019-07-26 17:13:13 CEST, end at Mon 2020-11-23 09:17:34 CET. --
-Nov 22 11:22:19 zeroed sshd[1480]: Invalid user wqeqwe from 127.0.0.1 port 55818
-Nov 22 11:22:23 zeroed sshd[1480]: Failed password for invalid user wqeqwe from 127.0.0.1 port 55818 ssh2
-Nov 22 11:23:22 zeroed sshd[1769]: Invalid user wqeqwe1 from 127.0.0.1 port 55824
-Nov 22 11:23:24 zeroed sshd[1769]: Disconnecting invalid user wqeqwe1 127.0.0.1 port 55824: Too many authentication failures [preauth]
-Nov 22 11:23:24 zeroed sshd[1777]: Invalid user wqeqwe2 from 127.0.0.1 port 55826
-Nov 22 11:23:25 zeroed sshd[1777]: Disconnecting invalid user wqeqwe2 127.0.0.1 port 55826: Too many authentication failures [preauth]
-Nov 22 11:23:25 zeroed sshd[1780]: Invalid user wqeqwe3 from 127.0.0.1 port 55828
-Nov 22 11:23:26 zeroed sshd[1780]: Disconnecting invalid user wqeqwe3 127.0.0.1 port 55828: Too many authentication failures [preauth]
-Nov 22 11:23:26 zeroed sshd[1786]: Invalid user wqeqwe4 from 127.0.0.1 port 55830
-Nov 22 11:23:27 zeroed sshd[1786]: Failed password for invalid user wqeqwe4 from 127.0.0.1 port 55830 ssh2
-Nov 22 11:23:27 zeroed sshd[1786]: Disconnecting invalid user wqeqwe4 127.0.0.1 port 55830: Too many authentication failures [preauth]
-Nov 22 11:23:27 zeroed sshd[1791]: Invalid user wqeqwe5 from 127.0.0.1 port 55834
-Nov 22 11:23:27 zeroed sshd[1791]: Failed password for invalid user wqeqwe5 from 127.0.0.1 port 55834 ssh2
-`
-
-func TestSimJournalctlCat(t *testing.T) {
-	if os.Getenv("GO_WANT_TEST_OUTPUT") != "1" {
-		return
-	}
-	defer os.Exit(0)
-	fmt.Print(testjournalctl_output_1)
-}
-
-func TestSimJournalctlCatError(t *testing.T) {
-	if os.Getenv("GO_WANT_TEST_OUTPUT") != "1" {
-		return
-	}
-	defer os.Exit(0)
-	fmt.Print("this is a single line being produced")
-	log.Warningf("this is an error message")
-}
-
-func TestSimJournalctlCatOneLine(t *testing.T) {
-	if os.Getenv("GO_WANT_TEST_OUTPUT") != "1" {
-		return
-	}
-	defer os.Exit(0)
-	fmt.Print("this is a single line being produced")
-}
-
-func TestJournaldTail(t *testing.T) {
-	tests := []struct {
-		cfg          DataSourceCfg
-		config_error string
-		read_error   string
-		tomb_error   string
-		lines        int
-	}{
-		{ //missing filename(s)
-			cfg: DataSourceCfg{
-				Mode: TAIL_MODE,
-			},
-			config_error: "journalctl_filter shouldn't be empty",
-		},
-		{ //bad mode
-			cfg: DataSourceCfg{
-				Mode:              "ratatata",
-				JournalctlFilters: []string{"-test.run=DoesNotExist", "--"},
-			},
-			/*here would actually be the journalctl error message on bad args, but you get the point*/
-			config_error: "unknown mode 'ratatata' for journald source",
-		},
-		{ //wrong arguments
-			cfg: DataSourceCfg{
-				Mode:              TAIL_MODE,
-				JournalctlFilters: []string{"--this-is-bad-option", "--"},
-			},
-			/*here would actually be the journalctl error message on bad args, but you get the point*/
-			tomb_error: "flag provided but not defined: -this-is-bad-option",
-		},
-	}
-
-	//we're actually using tests to do this, hold my beer and watch this
-	JOURNALD_CMD = os.Args[0]
-	JOURNALD_DEFAULT_TAIL_ARGS = []string{}
-
-	for tidx, test := range tests {
-		journalSrc := new(JournaldSource)
-		err := journalSrc.Configure(test.cfg)
-		if test.config_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", err), test.config_error)
-			log.Infof("expected config error ok : %s", test.config_error)
-			continue
-		} else {
-			if err != nil {
-				t.Fatalf("%d/%d unexpected config error %s", tidx, len(tests), err)
-			}
-		}
-
-		assert.Equal(t, journalSrc.Mode(), test.cfg.Mode)
-
-		//this tells our fake tests to produce data
-		journalSrc.Cmd.Env = []string{"GO_WANT_TEST_OUTPUT=1"}
-
-		out := make(chan types.Event)
-		tomb := tomb.Tomb{}
-		count := 0
-
-		//start consuming the data before we start the prog, so that chan isn't full
-		go func() {
-			for {
-				select {
-				case <-out:
-					count++
-				case <-time.After(1 * time.Second):
-					return
-				}
-			}
-		}()
-
-		err = journalSrc.StartReading(out, &tomb)
-		if test.read_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", err), test.read_error)
-			log.Infof("expected read error ok : %s", test.read_error)
-			continue
-		} else {
-			if err != nil {
-				t.Fatalf("%d/%d unexpected read error %s", tidx, len(tests), err)
-			}
-		}
-
-		time.Sleep(2 * time.Second)
-		log.Printf("now let's check number of lines & errors")
-		if count != test.lines {
-			t.Fatalf("%d/%d expected %d line read, got %d", tidx, len(tests), test.lines, count)
-		}
-
-		if test.tomb_error != "" {
-			assert.Contains(t, fmt.Sprintf("%s", tomb.Err()), test.tomb_error)
-			log.Infof("expected tomb error ok : %s", test.read_error)
-			continue
-		} else {
-			if tomb.Err() != nil {
-				t.Fatalf("%d/%d unexpected tomb error %s", tidx, len(tests), tomb.Err())
-			}
-		}
-
-	}
-}
-
-func TestJournaldSimple(t *testing.T) {
-	JOURNALD_CMD = os.Args[0]
-	JOURNALD_DEFAULT_TAIL_ARGS = []string{}
-	jBaseCfg := DataSourceCfg{
-		JournalctlFilters: []string{"-test.run=TestSimJournalctlCat", "--"},
-		Mode:              CAT_MODE,
-	}
-
-	journalSrc := new(JournaldSource)
-	err := journalSrc.Configure(jBaseCfg)
-	if err != nil {
-		t.Fatalf("configuring journalctl : %s", err)
-	}
-	journalSrc.Cmd.Env = []string{"GO_WANT_TEST_OUTPUT=1"}
-
-	out := make(chan types.Event)
-	tomb := tomb.Tomb{}
-	count := 0
-
-	//start the reading : it doesn't give hand back before it's done
-	err = journalSrc.StartReading(out, &tomb)
-	if err != nil {
-		t.Fatalf("unexpected read error %s", err)
-	}
-
-RLOOP:
-	for {
-		select {
-		case <-out:
-			count++
-		case <-time.After(1 * time.Second):
-			break RLOOP
-		}
-	}
-	//we expect 14 lines to be read
-	assert.Equal(t, 14, count)
-
-}
-
-func TestJournalctlKill(t *testing.T) {
-	cfg := DataSourceCfg{
-		Mode:              CAT_MODE,
-		JournalctlFilters: []string{"-test.run=TestSimJournalctlCatOneLine", "--"},
-	}
-	//we're actually using tests to do this, hold my beer and watch this
-	JOURNALD_CMD = os.Args[0]
-	JOURNALD_DEFAULT_TAIL_ARGS = []string{}
-
-	log.SetLevel(log.TraceLevel)
-	journalSrc := new(JournaldSource)
-	err := journalSrc.Configure(cfg)
-	if err != nil {
-		t.Fatalf("unexpected config error %s", err)
-	}
-	journalSrc.Cmd.Env = []string{"GO_WANT_TEST_OUTPUT=1"}
-
-	out := make(chan types.Event)
-	tb := tomb.Tomb{}
-
-	err = journalSrc.StartReading(out, &tb)
-	if err != nil {
-		t.Fatalf("unexpected read error %s", err)
-	}
-	time.Sleep(1 * time.Second)
-	if tb.Err() != tomb.ErrStillAlive {
-		t.Fatalf("unexpected tomb error %s (should be alive)", tb.Err())
-	}
-	//kill it :>
-	tb.Kill(nil)
-	time.Sleep(1 * time.Second)
-	if tb.Err() != nil {
-		t.Fatalf("unexpected tomb error %s (should be dead)", tb.Err())
-	}
-
-}

pkg/acquisition/modules/file/file.go

@@ -0,0 +1,358 @@
+package fileacquisition
+
+import (
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/fsnotify/fsnotify"
+	"github.com/nxadm/tail"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/tomb.v2"
+	"gopkg.in/yaml.v2"
+)
+
+type FileConfiguration struct {
+	Filenames                         []string
+	Filename                          string
+	ForceInotify                      bool `yaml:"force_inotify"`
+	configuration.DataSourceCommonCfg `yaml:",inline"`
+}
+
+type FileSource struct {
+	config             FileConfiguration
+	watcher            *fsnotify.Watcher
+	watchedDirectories map[string]bool
+	tails              map[string]bool
+	logger             *log.Entry
+	files              []string
+}
+
+func (f *FileSource) SupportedDSN() []string {
+	return []string{"file://"}
+}
+
+func (f *FileSource) Configure(Config []byte, logger *log.Entry) error {
+	fileConfig := FileConfiguration{}
+	f.logger = logger
+	f.watchedDirectories = make(map[string]bool)
+	f.tails = make(map[string]bool)
+	err := yaml.UnmarshalStrict(Config, &fileConfig)
+	if err != nil {
+		return errors.Wrap(err, "Cannot parse FileAcquisition configuration")
+	}
+	f.logger.Tracef("FileAcquisition configuration: %+v", fileConfig)
+	if len(fileConfig.Filename) != 0 {
+		fileConfig.Filenames = append(fileConfig.Filenames, fileConfig.Filename)
+	}
+	if len(fileConfig.Filenames) == 0 {
+		return fmt.Errorf("no filename or filenames configuration provided")
+	}
+	f.config = fileConfig
+	if f.config.Mode == "" {
+		f.config.Mode = configuration.TAIL_MODE
+	}
+	if f.config.Mode != configuration.CAT_MODE && f.config.Mode != configuration.TAIL_MODE {
+		return fmt.Errorf("unsupported mode %s for file source", f.config.Mode)
+	}
+	f.watcher, err = fsnotify.NewWatcher()
+	if err != nil {
+		return errors.Wrapf(err, "Could not create fsnotify watcher")
+	}
+	f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config)
+	for _, pattern := range f.config.Filenames {
+		if f.config.ForceInotify {
+			directory := path.Dir(pattern)
+			f.logger.Infof("Force add watch on %s", directory)
+			if !f.watchedDirectories[directory] {
+				err = f.watcher.Add(directory)
+				if err != nil {
+					f.logger.Errorf("Could not create watch on directory %s : %s", directory, err)
+					continue
+				}
+				f.watchedDirectories[directory] = true
+			}
+		}
+		files, err := filepath.Glob(pattern)
+		if err != nil {
+			return errors.Wrap(err, "Glob failure")
+		}
+		if len(files) == 0 {
+			f.logger.Infof("No matching files for pattern %s", pattern)
+			continue
+		}
+		f.logger.Infof("Will read %d files", len(files))
+		for _, file := range files {
+			if files[0] != pattern && f.config.Mode == configuration.TAIL_MODE { //we have a glob pattern
+				directory := path.Dir(file)
+				if !f.watchedDirectories[directory] {
+
+					err = f.watcher.Add(directory)
+					if err != nil {
+						f.logger.Errorf("Could not create watch on directory %s : %s", directory, err)
+						continue
+					}
+					f.watchedDirectories[directory] = true
+				}
+			}
+			f.logger.Infof("Adding file %s to filelist", file)
+			f.files = append(f.files, file)
+		}
+	}
+	return nil
+}
+
+func (f *FileSource) ConfigureByDSN(dsn string, labelType string, logger *log.Entry) error {
+	if !strings.HasPrefix(dsn, "file://") {
+		return fmt.Errorf("invalid DSN %s for file source, must start with file://", dsn)
+	}
+
+	pattern := strings.TrimPrefix(dsn, "file://")
+
+	if len(pattern) == 0 {
+		return fmt.Errorf("empty file:// DSN")
+	}
+
+	f.logger = logger
+	f.config = FileConfiguration{}
+	f.config.Labels = map[string]string{"type": labelType}
+	f.config.Mode = configuration.CAT_MODE
+
+	f.logger.Debugf("Will try pattern %s", pattern)
+	files, err := filepath.Glob(pattern)
+	if err != nil {
+		return errors.Wrap(err, "Glob failure")
+	}
+
+	if len(files) == 0 {
+		return fmt.Errorf("no matching files for pattern %s", pattern)
+	}
+
+	f.logger.Infof("Will read %d files", len(files))
+
+	for _, file := range files {
+		f.logger.Infof("Adding file %s to filelist", file)
+		f.files = append(f.files, file)
+	}
+	return nil
+}
+
+func (f *FileSource) GetMode() string {
+	return f.config.Mode
+}
+
+//SupportedModes returns the supported modes by the acquisition module
+func (f *FileSource) SupportedModes() []string {
+	return []string{configuration.TAIL_MODE, configuration.CAT_MODE}
+}
+
+//OneShotAcquisition reads a set of file and returns when done
+func (f *FileSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	f.logger.Debug("In oneshot")
+	for _, file := range f.files {
+		fi, err := os.Stat(file)
+		if err != nil {
+			return fmt.Errorf("could not stat file %s : %w", file, err)
+		}
+		if fi.IsDir() {
+			f.logger.Warnf("%s is a directory, ignoring it.", file)
+			continue
+		}
+		f.logger.Infof("reading %s at once", file)
+		err = f.readFile(file, out, t)
+		if err != nil {
+			return err
+		}
+
+	}
+	return nil
+}
+
+func (f *FileSource) GetMetrics() []prometheus.Collector {
+	return nil
+}
+
+func (f *FileSource) GetName() string {
+	return "file"
+}
+
+func (f *FileSource) CanRun() error {
+	return nil
+}
+
+func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	f.logger.Debug("Starting live acquisition")
+	t.Go(func() error {
+		return f.monitorNewFiles(out, t)
+	})
+	for _, file := range f.files {
+		tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}})
+		if err != nil {
+			f.logger.Errorf("Could not start tailing file %s : %s", file, err)
+			continue
+		}
+		f.tails[file] = true
+		t.Go(func() error {
+			defer types.CatchPanic("crowdsec/acquis/file/live/fsnotify")
+			return f.tailFile(out, t, tail)
+		})
+	}
+	return nil
+}
+
+func (f *FileSource) Dump() interface{} {
+	return f
+}
+
+func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
+	for {
+		select {
+		case event, ok := <-f.watcher.Events:
+			if !ok {
+				return nil
+			}
+			if event.Op&fsnotify.Create == fsnotify.Create {
+				fi, err := os.Stat(event.Name)
+				if err != nil {
+					f.logger.Errorf("Could not stat() new file %s, ignoring it : %s", event.Name, err)
+				}
+				if fi.IsDir() {
+					continue
+				}
+				f.logger.Infof("Detected new file %s", event.Name)
+				matched := false
+				for _, pattern := range f.config.Filenames {
+					f.logger.Debugf("Matching %s with %s", pattern, event.Name)
+					matched, err = path.Match(pattern, event.Name)
+					if err != nil {
+						f.logger.Errorf("Could not match pattern : %s", err)
+						continue
+					}
+					if matched {
+						break
+					}
+				}
+				if !matched {
+					continue
+				}
+				if f.tails[event.Name] {
+					//we already have a tail on it, do not start a new one
+					f.logger.Debugf("Already tailing file %s, not creating a new tail", event.Name)
+					break
+				}
+				//Slightly different parameters for Location, as we want to read the first lines of the newly created file
+				tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: true, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}})
+				if err != nil {
+					f.logger.Errorf("Could not start tailing file %s : %s", event.Name, err)
+					break
+				}
+				f.tails[event.Name] = true
+				t.Go(func() error {
+					defer types.CatchPanic("crowdsec/acquis/tailfile")
+					return f.tailFile(out, t, tail)
+				})
+			}
+		case err, ok := <-f.watcher.Errors:
+			if !ok {
+				return nil
+			}
+			f.logger.Errorf("Error while monitoring folder: %s", err)
+		}
+	}
+}
+
+func (f *FileSource) tailFile(out chan types.Event, t *tomb.Tomb, tail *tail.Tail) error {
+	//lint:ignore SA1015 as it is an infinite loop
+	timeout := time.Tick(1 * time.Second)
+	f.logger.Debugf("-> Starting tail of %s", tail.Filename)
+	for {
+		l := types.Line{}
+		select {
+		case <-t.Dying():
+			f.logger.Infof("File datasource %s stopping", tail.Filename)
+			if err := tail.Stop(); err != nil {
+				f.logger.Errorf("error in stop : %s", err)
+			}
+		case <-tail.Tomb.Dying(): //our tailer is dying
+			f.logger.Warningf("File reader of %s died", tail.Filename)
+			t.Kill(fmt.Errorf("dead reader for %s", tail.Filename))
+			return fmt.Errorf("reader for %s is dead", tail.Filename)
+		case line := <-tail.Lines:
+			if line == nil {
+				f.logger.Debugf("Nil line")
+				return fmt.Errorf("tail for %s is empty", tail.Filename)
+			}
+			if line.Err != nil {
+				log.Warningf("fetch error : %v", line.Err)
+				return line.Err
+			}
+			if line.Text == "" { //skip empty lines
+				continue
+			}
+			//FIXME: prometheus metrics
+			//ReaderHits.With(prometheus.Labels{"source": tail.Filename}).Inc()
+
+			l.Raw = line.Text
+			l.Labels = f.config.Labels
+			l.Time = line.Time
+			l.Src = tail.Filename
+			l.Process = true
+			//we're tailing, it must be real time logs
+			f.logger.Debugf("pushing %+v", l)
+			out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
+		case <-timeout:
+			//time out, shall we do stuff ?
+			f.logger.Trace("timeout")
+		}
+	}
+}
+
+func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tomb) error {
+	var scanner *bufio.Scanner
+
+	fd, err := os.Open(filename)
+
+	if err != nil {
+		return errors.Wrapf(err, "failed opening %s", filename)
+	}
+	defer fd.Close()
+
+	if strings.HasSuffix(filename, ".gz") {
+		gz, err := gzip.NewReader(fd)
+		if err != nil {
+			f.logger.Errorf("Failed to read gz file: %s", err)
+			return errors.Wrapf(err, "failed to read gz %s", filename)
+		}
+		defer gz.Close()
+		scanner = bufio.NewScanner(gz)
+
+	} else {
+		scanner = bufio.NewScanner(fd)
+	}
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		f.logger.Debugf("line %s", scanner.Text())
+		l := types.Line{}
+		l.Raw = scanner.Text()
+		l.Time = time.Now()
+		l.Src = filename
+		l.Labels = f.config.Labels
+		l.Process = true
+
+		//we're reading logs at once, it must be time-machine buckets
+		out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.TIMEMACHINE}
+	}
+	t.Kill(nil)
+	return nil
+}

pkg/acquisition/modules/file/file_reader_test.go

@@ -0,0 +1,139 @@
+package fileacquisition
+
+import (
+	"os"
+	"testing"
+
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/tomb.v2"
+)
+
+func TestBadConfiguration(t *testing.T) {
+	tests := []struct {
+		config      string
+		expectedErr string
+	}{
+		{
+			config:      `foobar: asd.log`,
+			expectedErr: "line 1: field foobar not found in type file_acquisition.FileConfiguration",
+		},
+		{
+			config:      `mode: tail`,
+			expectedErr: "no filename or filenames configuration provided",
+		},
+		{
+			config:      `filename: "[asd-.log"`,
+			expectedErr: "Glob failure: syntax error in pattern",
+		},
+	}
+
+	subLogger := log.WithFields(log.Fields{
+		"type": "file",
+	})
+	for _, test := range tests {
+		f := FileSource{}
+		err := f.Configure([]byte(test.config), subLogger)
+		assert.Contains(t, err.Error(), test.expectedErr)
+	}
+}
+
+func TestConfigureDSN(t *testing.T) {
+	tests := []struct {
+		dsn         string
+		expectedErr string
+	}{
+		{
+			dsn:         "asd://",
+			expectedErr: "invalid DSN asd:// for file source, must start with file://",
+		},
+		{
+			dsn:         "file://",
+			expectedErr: "empty file:// DSN",
+		},
+		{
+			dsn:         "file:///etc/passwd",
+			expectedErr: "",
+		},
+	}
+	subLogger := log.WithFields(log.Fields{
+		"type": "file",
+	})
+	for _, test := range tests {
+		f := FileSource{}
+		err := f.ConfigureByDSN(test.dsn, "testtype", subLogger)
+		if test.expectedErr != "" {
+			assert.Contains(t, err.Error(), test.expectedErr)
+		} else {
+			assert.Equal(t, err, nil)
+		}
+	}
+}
+
+func TestOneShot(t *testing.T) {
+	tests := []struct {
+		config         string
+		expectedErr    string
+		expectedOutput string
+	}{
+		{
+			config: `
+mode: cat
+filename: /etc/shadow`,
+			expectedErr:    "failed opening /etc/shadow: open /etc/shadow: permission denied",
+			expectedOutput: "",
+		},
+		{
+			config: `
+mode: cat
+filename: /`,
+			expectedErr:    "",
+			expectedOutput: "/ is a directory, ignoring it",
+		},
+	}
+
+	logger, hook := test.NewNullLogger()
+	logger.SetLevel(log.WarnLevel)
+	subLogger := logger.WithFields(log.Fields{
+		"type": "file",
+	})
+	tomb := tomb.Tomb{}
+	out := make(chan types.Event)
+
+	for _, test := range tests {
+		f := FileSource{}
+		err := f.Configure([]byte(test.config), subLogger)
+		if err != nil {
+			t.Fatalf("Unexpected error : %s", err)
+		}
+		err = f.OneShotAcquisition(out, &tomb)
+		if test.expectedErr != "" {
+			assert.Contains(t, err.Error(), test.expectedErr)
+		}
+		if test.expectedOutput != "" {
+			assert.Contains(t, hook.LastEntry().Message, test.expectedOutput)
+			continue
+		}
+	}
+}
+
+func TestLiveAcquisition(t *testing.T) {
+
+}
+
+func setup() {
+
+}
+
+func teardown() {
+
+}
+
+func TestMain(m *testing.M) {
+	setup()
+	code := m.Run()
+	teardown()
+	os.Exit(code)
+}

pkg/acquisition/modules/journalctl/journalctl_reader.go

@@ -0,0 +1,203 @@
+package journalctlacquisition
+
+import (
+	"bufio"
+	"fmt"
+	"net/url"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+
+	"gopkg.in/tomb.v2"
+	"gopkg.in/yaml.v2"
+)
+
+type JournalCtlConfiguration struct {
+	configuration.DataSourceCommonCfg `yaml:",inline"`
+	Filters                           []string `yaml:"journalctl_filter"`
+}
+
+type JournalCtlSource struct {
+	config JournalCtlConfiguration
+	logger *log.Entry
+	src    string
+	args   []string
+}
+
+const journalctlCmd string = "journalctl"
+
+var (
+	journalctlArgsOneShot  = []string{""}
+	journalctlArgstreaming = []string{"--follow"}
+)
+
+func (j *JournalCtlSource) runJournalCtl(out chan types.Event, t *tomb.Tomb) error {
+	cmd := exec.Command(journalctlCmd, j.args...)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf("could not get journalctl stdout: %s", err)
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return fmt.Errorf("could not get journalctl stderr: %s", err)
+	}
+
+	readErr := make(chan error)
+
+	j.logger.Debugf("Running journalctl command: %s %s", cmd.Path, cmd.Args)
+	err = cmd.Start()
+	if err != nil {
+		j.logger.Errorf("could not start journalctl command : %s", err)
+		return err
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(stderr)
+		if scanner == nil {
+			readErr <- fmt.Errorf("failed to create stderr scanner")
+			return
+		}
+		for scanner.Scan() {
+			txt := scanner.Text()
+			j.logger.Warningf("got stderr message : %s", txt)
+			readErr <- fmt.Errorf(txt)
+		}
+	}()
+
+	go func() {
+		scanner := bufio.NewScanner(stdout)
+		if scanner == nil {
+			readErr <- fmt.Errorf("failed to create stdout scanner")
+			return
+		}
+		for scanner.Scan() {
+			l := types.Line{}
+			//ReaderHits.With(prometheus.Labels{"source": j.SrcName}).Inc()
+			l.Raw = scanner.Text()
+			j.logger.Debugf("getting one line : %s", l.Raw)
+			l.Labels = j.config.Labels
+			l.Time = time.Now()
+			l.Src = j.src
+			l.Process = true
+			evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: leaky.LIVE}
+			out <- evt
+		}
+		j.logger.Debugf("finished reading from journalctl")
+		if err := scanner.Err(); err != nil {
+			j.logger.Debugf("got an error while reading %s : %s", j.src, err)
+			readErr <- err
+			return
+		}
+		readErr <- nil
+	}()
+
+	for {
+		select {
+		case <-t.Dying():
+			j.logger.Debugf("journalctl datasource %s stopping", j.src)
+			return nil
+		case err := <-readErr:
+			j.logger.Debugf("the subroutine returned, leave as well")
+			if err != nil {
+				j.logger.Warningf("journalctl reader error : %s", err)
+				t.Kill(err)
+			}
+			return err
+		}
+	}
+}
+
+func (j *JournalCtlSource) GetMetrics() []prometheus.Collector {
+	return nil
+}
+
+func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry) error {
+	config := JournalCtlConfiguration{}
+	j.logger = logger
+	err := yaml.UnmarshalStrict(yamlConfig, &config)
+	if err != nil {
+		return errors.Wrap(err, "Cannot parse JournalCtlSource configuration")
+	}
+	if config.Mode == "" {
+		config.Mode = configuration.TAIL_MODE
+	}
+	var args []string
+	if config.Mode == configuration.TAIL_MODE {
+		args = journalctlArgstreaming
+	} else {
+		args = journalctlArgsOneShot
+	}
+	j.args = append(args, config.Filters...)
+	j.src = fmt.Sprintf("journalctl-%s", strings.Join(config.Filters, "."))
+	j.config = config
+	return nil
+}
+
+func (j *JournalCtlSource) ConfigureByDSN(dsn string, labelType string, logger *log.Entry) error {
+	j.logger = logger
+	j.config = JournalCtlConfiguration{}
+	j.config.Mode = configuration.CAT_MODE
+	j.config.Labels = map[string]string{"type": labelType}
+
+	//format for the DSN is : journalctl://filters=FILTER1&filters=FILTER2
+	if !strings.HasPrefix(dsn, "journalctl://") {
+		return fmt.Errorf("invalid DSN %s for journalctl source, must start with journalctl://", dsn)
+	}
+
+	qs := strings.TrimPrefix(dsn, "journalctl://")
+	if len(qs) == 0 {
+		return fmt.Errorf("empty journalctl:// DSN")
+	}
+
+	params, err := url.ParseQuery(qs)
+	if err != nil {
+		return fmt.Errorf("could not parse journalctl DSN : %s", err)
+	}
+	for key, value := range params {
+		if key != "filters" {
+			return fmt.Errorf("unsupported key %s in journalctl DSN", key)
+		}
+		j.config.Filters = append(j.config.Filters, value...)
+	}
+	j.args = append(j.args, j.config.Filters...)
+	return nil
+}
+
+func (j *JournalCtlSource) GetMode() string {
+	return j.config.Mode
+}
+
+func (j *JournalCtlSource) GetName() string {
+	return "journalctl"
+}
+
+func (j *JournalCtlSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	t.Go(func() error {
+		defer types.CatchPanic("crowdsec/acquis/journalctl/oneshot")
+		return j.runJournalCtl(out, t)
+	})
+	return nil
+}
+
+func (j *JournalCtlSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
+	t.Go(func() error {
+		defer types.CatchPanic("crowdsec/acquis/journalctl/streaming")
+		return j.runJournalCtl(out, t)
+	})
+	return nil
+}
+func (j *JournalCtlSource) CanRun() error {
+	//TODO: add a more precise check on version or something ?
+	_, err := exec.LookPath(journalctlCmd)
+	return err
+}
+func (j *JournalCtlSource) Dump() interface{} {
+	return j
+}

pkg/acquisition/modules/journalctl/journalctl_reader_test.go

@@ -0,0 +1 @@
+package journalctlacquisition

pkg/acquisition/test_files/backward_compat.yaml

@@ -0,0 +1,15 @@
+filename: /tmp/test.log
+labels:
+  type: syslog
+---
+filenames:
+ - /tmp/test*.log
+labels:
+  type: syslog
+---
+# to be uncommented when we reimplement back journalctl
+# journalctl_filter:
+#  - "_SYSTEMD_UNIT=ssh.service"
+# labels:
+#   type: syslog
+---

pkg/acquisition/test_files/bad_filetype.yaml

@@ -0,0 +1,5 @@
+type: file
+filenames: /tmp/tltlt.log #it should be an array
+labels:
+  type: syslog
+  

pkg/acquisition/test_files/bad_source.yaml

@@ -0,0 +1,4 @@
+source: does_not_exist
+labels:
+  type: syslog
+foobar: toto

pkg/acquisition/test_files/badyaml.yaml

@@ -0,0 +1 @@
+<aaaa

pkg/acquisition/test_files/basic_filemode.yaml

@@ -0,0 +1,11 @@
+#type: file
+filename: /tmp/test.log
+labels:
+  type: syslog
+---
+#type: file
+filenames:
+ - /tmp/test*.log
+labels:
+  type: syslog
+---

pkg/acquisition/test_files/emptyitem.yaml

@@ -0,0 +1 @@
+

pkg/acquisition/test_files/missing_labels.yaml

@@ -0,0 +1,2 @@
+type: file
+filename: /tmp/test.log

pkg/acquisition/tests/acquis_test.yaml

@@ -1,5 +0,0 @@
-filenames:
-  - ./tests/test.log
-mode: tail
-labels:
-  type: my_test_log

pkg/acquisition/tests/test.log

@@ -1 +0,0 @@
-one log line