beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto detect if reading logs or storing sqlite db on a network share (#2241)

Browse source
This commit is contained in:
blotus 2024-03-07 14:04:50 +01:00 committed by GitHub
parent 98560d0cf5
commit 5731491b4e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 251 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
pkg/acquisition/modules/file/file.go
View file

@ -38,9 +38,9 @@ type FileConfiguration struct {
Filenames []string
ExcludeRegexps []string `yaml:"exclude_regexps"`
Filename string
ForceInotify bool `yaml:"force_inotify"`
MaxBufferSize int `yaml:"max_buffer_size"`
PollWithoutInotify bool `yaml:"poll_without_inotify"`
ForceInotify bool `yaml:"force_inotify"`
MaxBufferSize int `yaml:"max_buffer_size"`
PollWithoutInotify *bool `yaml:"poll_without_inotify"`
configuration.DataSourceCommonCfg `yaml:",inline"`
}
@ -330,7 +330,22 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er
continue
}
tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())})
inotifyPoll := true
if f.config.PollWithoutInotify != nil {
inotifyPoll = *f.config.PollWithoutInotify
} else {
networkFS, fsType, err := types.IsNetworkFS(file)
if err != nil {
f.logger.Warningf("Could not get fs type for %s : %s", file, err)
}
f.logger.Debugf("fs for %s is network: %t (%s)", file, networkFS, fsType)
if networkFS {
f.logger.Warnf("Disabling inotify poll on %s as it is on a network share. You can manually set poll_without_inotify to true to make this message disappear, or to false to enforce inotify poll", file)
inotifyPoll = false
}
}
tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())})
if err != nil {
f.logger.Errorf("Could not start tailing file %s : %s", file, err)
continue
@ -413,8 +428,27 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
f.logger.Errorf("unable to close %s : %s", event.Name, err)
continue
}
inotifyPoll := true
if f.config.PollWithoutInotify != nil {
inotifyPoll = *f.config.PollWithoutInotify
} else {
if f.config.PollWithoutInotify != nil {
inotifyPoll = *f.config.PollWithoutInotify
} else {
networkFS, fsType, err := types.IsNetworkFS(event.Name)
if err != nil {
f.logger.Warningf("Could not get fs type for %s : %s", event.Name, err)
}
f.logger.Debugf("fs for %s is network: %t (%s)", event.Name, networkFS, fsType)
if networkFS {
inotifyPoll = false
}
}
}
//Slightly different parameters for Location, as we want to read the first lines of the newly created file
tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}})
tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}})
if err != nil {
logger.Errorf("Could not start tailing file %s : %s", event.Name, err)
break

1
pkg/csconfig/api_test.go
View file

@ -194,6 +194,7 @@ func TestLoadAPIServer(t *testing.T) {
DbPath: "./testdata/test.db",
Type: "sqlite",
MaxOpenConns: ptr.Of(DEFAULT_MAX_OPEN_CONNS),
UseWal: ptr.Of(true), // autodetected
DecisionBulkSize: defaultDecisionBulkSize,
},
ConsoleConfigPath: DefaultConfigPath("console.yaml"),

36
pkg/csconfig/database.go
View file

@ -3,12 +3,15 @@ package csconfig
import (
"errors"
"fmt"
"path/filepath"
"time"
"entgo.io/ent/dialect"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const (
@ -69,6 +72,35 @@ func (c *Config) LoadDBConfig(inCli bool) error {
c.DbConfig.MaxOpenConns = ptr.Of(DEFAULT_MAX_OPEN_CONNS)
}
if !inCli && c.DbConfig.Type == "sqlite" {
if c.DbConfig.UseWal == nil {
dbDir := filepath.Dir(c.DbConfig.DbPath)
isNetwork, fsType, err := types.IsNetworkFS(dbDir)
if err != nil {
log.Warnf("unable to determine if database is on network filesystem: %s", err)
log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.")
return nil
}
if isNetwork {
log.Debugf("database is on network filesystem (%s), setting useWal to false", fsType)
c.DbConfig.UseWal = ptr.Of(false)
} else {
log.Debugf("database is on local filesystem (%s), setting useWal to true", fsType)
c.DbConfig.UseWal = ptr.Of(true)
}
} else if *c.DbConfig.UseWal {
dbDir := filepath.Dir(c.DbConfig.DbPath)
isNetwork, fsType, err := types.IsNetworkFS(dbDir)
if err != nil {
log.Warnf("unable to determine if database is on network filesystem: %s", err)
return nil
}
if isNetwork {
log.Warnf("database seems to be stored on a network share (%s), but useWal is set to true. Proceed at your own risk.", fsType)
}
}
}
if c.DbConfig.DecisionBulkSize == 0 {
log.Tracef("No decision_bulk_size value provided, using default value of %d", defaultDecisionBulkSize)
c.DbConfig.DecisionBulkSize = defaultDecisionBulkSize
@ -79,10 +111,6 @@ func (c *Config) LoadDBConfig(inCli bool) error {
c.DbConfig.DecisionBulkSize = maxDecisionBulkSize
}
if !inCli && c.DbConfig.Type == "sqlite" && c.DbConfig.UseWal == nil {
log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.")
}
return nil
}

7
pkg/csconfig/database_test.go
View file

@ -30,9 +30,10 @@ func TestLoadDBConfig(t *testing.T) {
},
},
expected: &DatabaseCfg{
Type: "sqlite",
DbPath: "./testdata/test.db",
MaxOpenConns: ptr.Of(10),
Type: "sqlite",
DbPath: "./testdata/test.db",
MaxOpenConns: ptr.Of(10),
UseWal: ptr.Of(true),
DecisionBulkSize: defaultDecisionBulkSize,
},
},

112
pkg/types/getfstype.go Normal file
View file

@ -0,0 +1,112 @@
//go:build !windows
package types
import (
"fmt"
"syscall"
)
// Generated with `man statfs | grep _MAGIC | awk '{split(tolower($1),a,"_"); print $2 ": \"" a[1] "\","}'`
// ext2/3/4 duplicates removed to just have ext4
// XIAFS removed as well
var fsTypeMapping map[int]string = map[int]string{
0xadf5: "adfs",
0xadff: "affs",
0x5346414f: "afs",
0x09041934: "anon",
0x0187: "autofs",
0x62646576: "bdevfs",
0x42465331: "befs",
0x1badface: "bfs",
0x42494e4d: "binfmtfs",
0xcafe4a11: "bpf",
0x9123683e: "btrfs",
0x73727279: "btrfs",
0x27e0eb: "cgroup",
0x63677270: "cgroup2",
0xff534d42: "cifs",
0x73757245: "coda",
0x012ff7b7: "coh",
0x28cd3d45: "cramfs",
0x64626720: "debugfs",
0x1373: "devfs",
0x1cd1: "devpts",
0xf15f: "ecryptfs",
0xde5e81e4: "efivarfs",
0x00414a53: "efs",
0x137d: "ext",
0xef51: "ext2",
0xef53: "ext4",
0xf2f52010: "f2fs",
0x65735546: "fuse",
0xbad1dea: "futexfs",
0x4244: "hfs",
0x00c0ffee: "hostfs",
0xf995e849: "hpfs",
0x958458f6: "hugetlbfs",
0x9660: "isofs",
0x72b6: "jffs2",
0x3153464a: "jfs",
0x137f: "minix",
0x138f: "minix",
0x2468: "minix2",
0x2478: "minix2",
0x4d5a: "minix3",
0x19800202: "mqueue",
0x4d44: "msdos",
0x11307854: "mtd",
0x564c: "ncp",
0x6969: "nfs",
0x3434: "nilfs",
0x6e736673: "nsfs",
0x5346544e: "ntfs",
0x7461636f: "ocfs2",
0x9fa1: "openprom",
0x794c7630: "overlayfs",
0x50495045: "pipefs",
0x9fa0: "proc",
0x6165676c: "pstorefs",
0x002f: "qnx4",
0x68191122: "qnx6",
0x858458f6: "ramfs",
0x52654973: "reiserfs",
0x7275: "romfs",
0x73636673: "securityfs",
0xf97cff8c: "selinux",
0x43415d53: "smack",
0x517b: "smb",
0xfe534d42: "smb2",
0x534f434b: "sockfs",
0x73717368: "squashfs",
0x62656572: "sysfs",
0x012ff7b6: "sysv2",
0x012ff7b5: "sysv4",
0x01021994: "tmpfs",
0x74726163: "tracefs",
0x15013346: "udf",
0x00011954: "ufs",
0x9fa2: "usbdevice",
0x01021997: "v9fs",
0xa501fcf5: "vxfs",
0xabba1974: "xenfs",
0x012ff7b4: "xenix",
0x58465342: "xfs",
}
func GetFSType(path string) (string, error) {
var buf syscall.Statfs_t
err := syscall.Statfs(path, &buf)
if err != nil {
return "", err
}
fsType, ok := fsTypeMapping[int(buf.Type)]
if !ok {
return "", fmt.Errorf("unknown fstype %d", buf.Type)
}
return fsType, nil
}

53
pkg/types/getfstype_windows.go Normal file
View file

@ -0,0 +1,53 @@
package types
import (
"path/filepath"
"syscall"
"unsafe"
)
func GetFSType(path string) (string, error) {
kernel32, err := syscall.LoadLibrary("kernel32.dll")
if err != nil {
return "", err
}
defer syscall.FreeLibrary(kernel32)
getVolumeInformation, err := syscall.GetProcAddress(kernel32, "GetVolumeInformationW")
if err != nil {
return "", err
}
// Convert relative path to absolute path
absPath, err := filepath.Abs(path)
if err != nil {
return "", err
}
// Get the root path of the volume
volumeRoot := filepath.VolumeName(absPath) + "\\"
volumeRootPtr, _ := syscall.UTF16PtrFromString(volumeRoot)
var (
fileSystemNameBuffer = make([]uint16, 260)
nFileSystemNameSize = uint32(len(fileSystemNameBuffer))
)
ret, _, err := syscall.SyscallN(getVolumeInformation,
uintptr(unsafe.Pointer(volumeRootPtr)),
0,
0,
0,
0,
0,
uintptr(unsafe.Pointer(&fileSystemNameBuffer[0])),
uintptr(nFileSystemNameSize),
0)
if ret == 0 {
return "", err
}
return syscall.UTF16ToString(fileSystemNameBuffer), nil
}

10
pkg/types/utils.go
View file

@ -3,6 +3,7 @@ package types
import (
"fmt"
"path/filepath"
"strings"
"time"
log "github.com/sirupsen/logrus"
@ -67,3 +68,12 @@ func ConfigureLogger(clog *log.Logger) error {
func UtcNow() time.Time {
return time.Now().UTC()
}
func IsNetworkFS(path string) (bool, string, error) {
fsType, err := GetFSType(path)
if err != nil {
return false, "", err
}
fsType = strings.ToLower(fsType)
return fsType == "nfs" || fsType == "cifs" || fsType == "smb" || fsType == "smb2", fsType, nil
}