0ct0pu5/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into support_data_download

Browse source
This commit is contained in:
Thibault "bui" Koechlin 2020-05-27 19:03:06 +02:00 • committed by GitHub
commit 373cd2ecd1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 98 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -62,6 +62,7 @@ else
@echo "Required golang version is $(REQUIRE_GOVERSION). The current one is $(CURRENT_GOVERSION). Exiting.."
@exit 1;
endif
@bash ./scripts/build_plugins.sh
cscli_static:

5
cmd/crowdsec-cli/main.go
View file

@ -117,7 +117,10 @@ API interaction:
rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")
rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.")
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.")
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "master", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to make branch hidden : %s", err)
}
cobra.OnInitialize(initConfig)
/*don't sort flags so we can enforce order*/
rootCmd.Flags().SortFlags = false

31
cmd/crowdsec/main.go
View file

@ -1,7 +1,6 @@
package main
import (
"fmt"
"strings"
"io/ioutil"
@ -19,7 +18,6 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v2"
)
@ -39,33 +37,6 @@ var (
lastProcessedItem time.Time /*keep track of last item timestamp in time-machine. it is used to GC buckets when we dump them.*/
)
func configureLogger(logMode string, logFolder string, logLevel log.Level) error {
/*Configure logs*/
if logMode == "file" {
log.SetOutput(&lumberjack.Logger{
Filename: logFolder + "/crowdsec.log",
MaxSize: 500, //megabytes
MaxBackups: 3,
MaxAge: 28, //days
Compress: true, //disabled by default
})
log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true})
} else if logMode != "stdout" {
return fmt.Errorf("log mode '%s' unknown", logMode)
}
log.Printf("setting loglevel to %s", logLevel)
log.SetLevel(logLevel)
log.SetFormatter(&log.TextFormatter{FullTimestamp: true})
if logLevel >= log.InfoLevel {
log.SetFormatter(&log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true})
}
if logLevel >= log.DebugLevel {
log.SetReportCaller(true)
}
return nil
}
func main() {
var (
err error
@ -92,7 +63,7 @@ func main() {
log.Fatalf(err.Error())
}
if err = configureLogger(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil {
if err = types.SetDefaultLoggerConfig(cConfig.LogMode, cConfig.LogFolder, cConfig.LogLevel); err != nil {
log.Fatal(err.Error())
}

2
config/plugins/backend/sqlite.yaml
View file

@ -1,5 +1,5 @@
name: sqlite
path: /var/lib/crowdsec/plugins/backend/sqlite.so
path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so
config:
db_path: /var/lib/crowdsec/data/crowdsec.db
flush: true

4
config/user.yaml
View file

@ -8,9 +8,9 @@ log_mode: stdout
log_level: info
profiling: false
sqlite_path: ${DATA}/crowdsec.db
apimode: true
apimode: false
daemon: false
prometheus: true
prometheus: false
#for prometheus agent / golang debugging
http_listen: 127.0.0.1:6060
plugin:

1
docs/getting_started/installation.md
View file

@ -65,6 +65,7 @@ You will need as well to configure your {{ref.acquis}} file to feed {{crowdsec.n
* [Go](https://golang.org/doc/install) v1.13+
* `git clone {{crowdsec.url}}`
* [jq](https://stedolan.github.io/jq/download/)
Go in {{crowdsec.name}} folder and build the binaries :

2
docs/references/output.md
View file

@ -62,7 +62,7 @@ config: <plugin_config> # in a form of key(string)/value(string)
For the plugin sqlite, here is its configuration file:
```yaml
name: sqlite
path: /var/lib/crowdsec/plugins/backend/sqlite.so
path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so
config:
db_path: /var/lib/crowdsec/data/crowdsec.db
flush: true

2
mkdocs.yml
View file

@ -129,7 +129,7 @@ extra:
plugins:
name: blockers
url: "https://hub.crowdsec.net/"
binpath: "/var/lib/crowdsec/plugins/"
binpath: "/usr/local/crowdsec/plugins/"
configpath: "/etc/crowdsec/plugins/"
metabase:
name: metabase

11
pkg/cwhub/hubMgmt.go
View file

@ -35,8 +35,9 @@ var Installdir = "/etc/crowdsec/"
var Hubdir = "/etc/crowdsec/cscli/hub/"
var Cfgdir = "/etc/crowdsec/cscli/"
var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/master/%s"
var HUB_INDEX_FILE = ".index.json"
var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s"
var HubIndexFile = ".index.json"
var HubBranch = "master"
type ItemVersion struct {
Digest string
@ -409,7 +410,7 @@ func UpdateHubIdx() error {
}
func DownloadHubIdx() ([]byte, error) {
req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE), nil)
req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
if err != nil {
log.Errorf("failed request : %s", err)
return nil, err
@ -421,7 +422,7 @@ func DownloadHubIdx() ([]byte, error) {
}
if resp.StatusCode != 200 {
log.Errorf("got code %d while requesting %s, abort", resp.StatusCode,
fmt.Sprintf(RawFileURLTemplate, HUB_INDEX_FILE))
fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
return nil, fmt.Errorf("bad http code")
}
defer resp.Body.Close()
@ -687,7 +688,7 @@ func DownloadItem(target Item, tdir string, overwrite bool, dataFolder string) (
}
//log.Infof("Downloading %s to %s", target.Name, tdir)
req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, target.RemotePath), nil)
req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
if err != nil {
log.Errorf("%s : request creation failed : %s", target.Name, err)
return target, err

8
pkg/leakybucket/manager.go
View file

@ -151,7 +151,7 @@ func LoadBuckets(files []string, dataFolder string) ([]BucketFactory, chan types
}
//check compat
if g.FormatVersion == "" {
log.Warningf("no version in %s : %s, assuming '1.0'", g.Name, f)
log.Debugf("no version in %s : %s, assuming '1.0'", g.Name, f)
g.FormatVersion = "1.0"
}
ok, err := cwversion.Statisfies(g.FormatVersion, cwversion.Constraint_scenario)
@ -195,8 +195,10 @@ func LoadBucketDir(dir string, dataFolder string) ([]BucketFactory, chan types.E
func LoadBucket(g *BucketFactory, dataFolder string) error {
var err error
if g.Debug {
var clog = log.New()
clog.SetFormatter(&log.TextFormatter{FullTimestamp: true})
var clog = logrus.New()
if err := types.ConfigureLogger(clog); err != nil {
log.Fatalf("While creating bucket-specific logger : %s", err)
}
clog.SetLevel(log.DebugLevel)
g.logger = clog.WithFields(log.Fields{
"cfg": g.BucketName,

17
pkg/parser/node.go
View file

@ -253,10 +253,15 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
//return false, nil
}
}
var groklabel string
if n.Grok.RegexpName == "" {
groklabel = fmt.Sprintf("%5.5s...", n.Grok.RegexpValue)
} else {
groklabel = n.Grok.RegexpName
}
grok := n.Grok.RunTimeRegexp.Parse(gstr)
if len(grok) > 0 {
clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", n.Grok.RegexpName, len(grok))
clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok))
//We managed to grok stuff, merged into parse
for k, v := range grok {
clog.Debugf("\t.Parsed['%s'] = '%s'", k, v)
@ -269,7 +274,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
}
} else {
//grok failed, node failed
clog.Debugf("+ Grok '%s' didn't return data on '%s'", n.Grok.RegexpName, gstr)
clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr)
//clog.Tracef("on '%s'", gstr)
NodeState = false
}
@ -337,6 +342,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error {
that will be used only for processing this node ;) */
if n.Debug {
var clog = logrus.New()
if err := types.ConfigureLogger(clog); err != nil {
log.Fatalf("While creating bucket-specific logger : %s", err)
}
clog.SetLevel(log.DebugLevel)
n.logger = clog.WithFields(log.Fields{
"id": n.rn,
@ -412,6 +420,9 @@ func (n *Node) compile(pctx *UnixParserCtx) error {
/* compile leafs if present */
if len(n.SuccessNodes) > 0 {
for idx := range n.SuccessNodes {
if n.SuccessNodes[idx].Name == "" {
n.SuccessNodes[idx].Name = fmt.Sprintf("child-%s", n.Name)
}
/*propagate debug/stats to child nodes*/
if !n.SuccessNodes[idx].Debug && n.Debug {
n.SuccessNodes[idx].Debug = true

2
pkg/parser/stage.go
View file

@ -81,7 +81,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) {
//check for empty bucket
if node.Name == "" && node.Description == "" && node.Author == "" {
log.Infof("Node has no name,author or description. Skipping.")
log.Infof("Node in %s has no name,author or description. Skipping.", stageFile.Filename)
continue
}
//check compat

53
pkg/types/utils.go
View file

@ -2,7 +2,12 @@ package types
import (
"encoding/binary"
"fmt"
"io"
"net"
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
)
func IP2Int(ip net.IP) uint32 {
@ -40,3 +45,51 @@ func LastAddress(n *net.IPNet) net.IP {
ip[2]|^n.Mask[2],
ip[3]|^n.Mask[3])
}
var logFormatter log.Formatter
var logOutput io.Writer
var logLevel log.Level
var logReportCaller bool
func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level) error {
/*Configure logs*/
if cfgMode == "file" {
logOutput = &lumberjack.Logger{
Filename: cfgFolder + "/crowdsec.log",
MaxSize: 500, //megabytes
MaxBackups: 3,
MaxAge: 28, //days
Compress: true, //disabled by default
}
log.SetOutput(logOutput)
} else if cfgMode != "stdout" {
return fmt.Errorf("log mode '%s' unknown", cfgMode)
}
logLevel = cfgLevel
log.SetLevel(logLevel)
if logLevel >= log.InfoLevel {
logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
log.SetFormatter(logFormatter)
}
if logLevel >= log.DebugLevel {
logReportCaller = true
log.SetReportCaller(true)
}
return nil
}
func ConfigureLogger(clog *log.Logger) error {
/*Configure logs*/
if logOutput != nil {
clog.SetOutput(logOutput)
}
if logReportCaller {
clog.SetReportCaller(true)
}
if logFormatter != nil {
clog.SetFormatter(logFormatter)
}
clog.SetLevel(logLevel)
return nil
}

12
wizard.sh
View file

@ -14,12 +14,13 @@ SILENT="false"
CROWDSEC_RUN_DIR="/var/run"
CROWDSEC_LIB_DIR="/var/lib/crowdsec"
CROWDSEC_USR_DIR="/usr/local/lib/crowdsec"
CROWDSEC_DATA_DIR="${CROWDSEC_LIB_DIR}/data"
CROWDSEC_PLUGIN_DIR="${CROWDSEC_LIB_DIR}/plugins"
CROWDSEC_PLUGIN_DIR="${CROWDSEC_USR_DIR}/plugins"
CROWDSEC_PLUGIN_BACKEND_DIR="${CROWDSEC_PLUGIN_DIR}/backend"
CROWDSEC_DB_PATH="${CROWDSEC_DATA_DIR}/crowdsec.db"
CROWDSEC_CONFIG_PATH="/etc/crowdsec"
CROWDSEC_CONFIG_PATH="${CROWDSEC_CONFIG_PATH}/config"
CROWDSEC_PATH="/etc/crowdsec"
CROWDSEC_CONFIG_PATH="${CROWDSEC_PATH}/config"
CROWDSEC_LOG_FILE="/var/log/crowdsec.log"
CROWDSEC_BACKEND_FOLDER="/etc/crowdsec/plugins/backend"
CSCLI_FOLDER="/etc/crowdsec/config/cscli"
@ -39,7 +40,7 @@ setup_cron_pull() {
}
PID_DIR="/var/run"
PID_DIR="${CROWDSEC_RUN_DIR}"
SYSTEMD_PATH_FILE="/etc/systemd/system/crowdsec.service"
PATTERNS_FOLDER="config/patterns"
@ -340,10 +341,11 @@ uninstall_crowdsec() {
systemctl stop crowdsec.service
${CSCLI_BIN} dashboard stop --remove
delete_bins
rm -rf ${CROWDSEC_CONFIG_PATH} || echo ""
rm -rf ${CROWDSEC_PATH} || echo ""
rm -f ${CROWDSEC_LOG_FILE} || echo ""
rm -f ${CROWDSEC_DB_PATH} || echo ""
rm -rf ${CROWDSEC_LIB_DIR} || echo ""
rm -rf ${CROWDSEC_USR_DIR} || echo ""
rm -f ${SYSTEMD_PATH_FILE} || echo ""
log_info "crowdsec successfully uninstalled"
}