From 5731491b4e0948e5011e47a378a14f9a86e46b40 Mon Sep 17 00:00:00 2001 From: blotus Date: Thu, 7 Mar 2024 14:04:50 +0100 Subject: [PATCH] Auto detect if reading logs or storing sqlite db on a network share (#2241) --- pkg/acquisition/modules/file/file.go | 44 +++++++++-- pkg/csconfig/api_test.go | 1 + pkg/csconfig/database.go | 36 ++++++++- pkg/csconfig/database_test.go | 7 +- pkg/types/getfstype.go | 112 +++++++++++++++++++++++++++ pkg/types/getfstype_windows.go | 53 +++++++++++++ pkg/types/utils.go | 10 +++ 7 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 pkg/types/getfstype.go create mode 100644 pkg/types/getfstype_windows.go diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 4ea9466d4..9ab418a84 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -38,9 +38,9 @@ type FileConfiguration struct { Filenames []string ExcludeRegexps []string `yaml:"exclude_regexps"` Filename string - ForceInotify bool `yaml:"force_inotify"` - MaxBufferSize int `yaml:"max_buffer_size"` - PollWithoutInotify bool `yaml:"poll_without_inotify"` + ForceInotify bool `yaml:"force_inotify"` + MaxBufferSize int `yaml:"max_buffer_size"` + PollWithoutInotify *bool `yaml:"poll_without_inotify"` configuration.DataSourceCommonCfg `yaml:",inline"` } @@ -330,7 +330,22 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er continue } - tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())}) + inotifyPoll := true + if f.config.PollWithoutInotify != nil { + inotifyPoll = *f.config.PollWithoutInotify + } else { + networkFS, fsType, err := types.IsNetworkFS(file) + if err != nil { + f.logger.Warningf("Could not get fs type for %s : %s", file, err) + } + f.logger.Debugf("fs for %s is network: %t (%s)", file, networkFS, fsType) + if networkFS { + f.logger.Warnf("Disabling inotify poll on %s as it is on a network share. You can manually set poll_without_inotify to true to make this message disappear, or to false to enforce inotify poll", file) + inotifyPoll = false + } + } + + tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())}) if err != nil { f.logger.Errorf("Could not start tailing file %s : %s", file, err) continue @@ -413,8 +428,27 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error { f.logger.Errorf("unable to close %s : %s", event.Name, err) continue } + + inotifyPoll := true + if f.config.PollWithoutInotify != nil { + inotifyPoll = *f.config.PollWithoutInotify + } else { + if f.config.PollWithoutInotify != nil { + inotifyPoll = *f.config.PollWithoutInotify + } else { + networkFS, fsType, err := types.IsNetworkFS(event.Name) + if err != nil { + f.logger.Warningf("Could not get fs type for %s : %s", event.Name, err) + } + f.logger.Debugf("fs for %s is network: %t (%s)", event.Name, networkFS, fsType) + if networkFS { + inotifyPoll = false + } + } + } + //Slightly different parameters for Location, as we want to read the first lines of the newly created file - tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}}) + tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}}) if err != nil { logger.Errorf("Could not start tailing file %s : %s", event.Name, err) break diff --git a/pkg/csconfig/api_test.go b/pkg/csconfig/api_test.go index b6febd4d4..463b7c1b2 100644 --- a/pkg/csconfig/api_test.go +++ b/pkg/csconfig/api_test.go @@ -194,6 +194,7 @@ func TestLoadAPIServer(t *testing.T) { DbPath: "./testdata/test.db", Type: "sqlite", MaxOpenConns: ptr.Of(DEFAULT_MAX_OPEN_CONNS), + UseWal: ptr.Of(true), // autodetected DecisionBulkSize: defaultDecisionBulkSize, }, ConsoleConfigPath: DefaultConfigPath("console.yaml"), diff --git a/pkg/csconfig/database.go b/pkg/csconfig/database.go index 2df220785..a7bc57eef 100644 --- a/pkg/csconfig/database.go +++ b/pkg/csconfig/database.go @@ -3,12 +3,15 @@ package csconfig import ( "errors" "fmt" + "path/filepath" "time" "entgo.io/ent/dialect" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/go-cs-lib/ptr" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -69,6 +72,35 @@ func (c *Config) LoadDBConfig(inCli bool) error { c.DbConfig.MaxOpenConns = ptr.Of(DEFAULT_MAX_OPEN_CONNS) } + if !inCli && c.DbConfig.Type == "sqlite" { + if c.DbConfig.UseWal == nil { + dbDir := filepath.Dir(c.DbConfig.DbPath) + isNetwork, fsType, err := types.IsNetworkFS(dbDir) + if err != nil { + log.Warnf("unable to determine if database is on network filesystem: %s", err) + log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.") + return nil + } + if isNetwork { + log.Debugf("database is on network filesystem (%s), setting useWal to false", fsType) + c.DbConfig.UseWal = ptr.Of(false) + } else { + log.Debugf("database is on local filesystem (%s), setting useWal to true", fsType) + c.DbConfig.UseWal = ptr.Of(true) + } + } else if *c.DbConfig.UseWal { + dbDir := filepath.Dir(c.DbConfig.DbPath) + isNetwork, fsType, err := types.IsNetworkFS(dbDir) + if err != nil { + log.Warnf("unable to determine if database is on network filesystem: %s", err) + return nil + } + if isNetwork { + log.Warnf("database seems to be stored on a network share (%s), but useWal is set to true. Proceed at your own risk.", fsType) + } + } + } + if c.DbConfig.DecisionBulkSize == 0 { log.Tracef("No decision_bulk_size value provided, using default value of %d", defaultDecisionBulkSize) c.DbConfig.DecisionBulkSize = defaultDecisionBulkSize @@ -79,10 +111,6 @@ func (c *Config) LoadDBConfig(inCli bool) error { c.DbConfig.DecisionBulkSize = maxDecisionBulkSize } - if !inCli && c.DbConfig.Type == "sqlite" && c.DbConfig.UseWal == nil { - log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.") - } - return nil } diff --git a/pkg/csconfig/database_test.go b/pkg/csconfig/database_test.go index a94602579..c7741baf0 100644 --- a/pkg/csconfig/database_test.go +++ b/pkg/csconfig/database_test.go @@ -30,9 +30,10 @@ func TestLoadDBConfig(t *testing.T) { }, }, expected: &DatabaseCfg{ - Type: "sqlite", - DbPath: "./testdata/test.db", - MaxOpenConns: ptr.Of(10), + Type: "sqlite", + DbPath: "./testdata/test.db", + MaxOpenConns: ptr.Of(10), + UseWal: ptr.Of(true), DecisionBulkSize: defaultDecisionBulkSize, }, }, diff --git a/pkg/types/getfstype.go b/pkg/types/getfstype.go new file mode 100644 index 000000000..4a54fc948 --- /dev/null +++ b/pkg/types/getfstype.go @@ -0,0 +1,112 @@ +//go:build !windows + +package types + +import ( + "fmt" + "syscall" +) + +// Generated with `man statfs | grep _MAGIC | awk '{split(tolower($1),a,"_"); print $2 ": \"" a[1] "\","}'` +// ext2/3/4 duplicates removed to just have ext4 +// XIAFS removed as well +var fsTypeMapping map[int]string = map[int]string{ + 0xadf5: "adfs", + 0xadff: "affs", + 0x5346414f: "afs", + 0x09041934: "anon", + 0x0187: "autofs", + 0x62646576: "bdevfs", + 0x42465331: "befs", + 0x1badface: "bfs", + 0x42494e4d: "binfmtfs", + 0xcafe4a11: "bpf", + 0x9123683e: "btrfs", + 0x73727279: "btrfs", + 0x27e0eb: "cgroup", + 0x63677270: "cgroup2", + 0xff534d42: "cifs", + 0x73757245: "coda", + 0x012ff7b7: "coh", + 0x28cd3d45: "cramfs", + 0x64626720: "debugfs", + 0x1373: "devfs", + 0x1cd1: "devpts", + 0xf15f: "ecryptfs", + 0xde5e81e4: "efivarfs", + 0x00414a53: "efs", + 0x137d: "ext", + 0xef51: "ext2", + 0xef53: "ext4", + 0xf2f52010: "f2fs", + 0x65735546: "fuse", + 0xbad1dea: "futexfs", + 0x4244: "hfs", + 0x00c0ffee: "hostfs", + 0xf995e849: "hpfs", + 0x958458f6: "hugetlbfs", + 0x9660: "isofs", + 0x72b6: "jffs2", + 0x3153464a: "jfs", + 0x137f: "minix", + 0x138f: "minix", + 0x2468: "minix2", + 0x2478: "minix2", + 0x4d5a: "minix3", + 0x19800202: "mqueue", + 0x4d44: "msdos", + 0x11307854: "mtd", + 0x564c: "ncp", + 0x6969: "nfs", + 0x3434: "nilfs", + 0x6e736673: "nsfs", + 0x5346544e: "ntfs", + 0x7461636f: "ocfs2", + 0x9fa1: "openprom", + 0x794c7630: "overlayfs", + 0x50495045: "pipefs", + 0x9fa0: "proc", + 0x6165676c: "pstorefs", + 0x002f: "qnx4", + 0x68191122: "qnx6", + 0x858458f6: "ramfs", + 0x52654973: "reiserfs", + 0x7275: "romfs", + 0x73636673: "securityfs", + 0xf97cff8c: "selinux", + 0x43415d53: "smack", + 0x517b: "smb", + 0xfe534d42: "smb2", + 0x534f434b: "sockfs", + 0x73717368: "squashfs", + 0x62656572: "sysfs", + 0x012ff7b6: "sysv2", + 0x012ff7b5: "sysv4", + 0x01021994: "tmpfs", + 0x74726163: "tracefs", + 0x15013346: "udf", + 0x00011954: "ufs", + 0x9fa2: "usbdevice", + 0x01021997: "v9fs", + 0xa501fcf5: "vxfs", + 0xabba1974: "xenfs", + 0x012ff7b4: "xenix", + 0x58465342: "xfs", +} + +func GetFSType(path string) (string, error) { + var buf syscall.Statfs_t + + err := syscall.Statfs(path, &buf) + + if err != nil { + return "", err + } + + fsType, ok := fsTypeMapping[int(buf.Type)] + if !ok { + return "", fmt.Errorf("unknown fstype %d", buf.Type) + } + + return fsType, nil +} diff --git a/pkg/types/getfstype_windows.go b/pkg/types/getfstype_windows.go new file mode 100644 index 000000000..03d8fffd4 --- /dev/null +++ b/pkg/types/getfstype_windows.go @@ -0,0 +1,53 @@ +package types + +import ( + "path/filepath" + "syscall" + "unsafe" +) + +func GetFSType(path string) (string, error) { + kernel32, err := syscall.LoadLibrary("kernel32.dll") + if err != nil { + return "", err + } + defer syscall.FreeLibrary(kernel32) + + getVolumeInformation, err := syscall.GetProcAddress(kernel32, "GetVolumeInformationW") + if err != nil { + return "", err + } + + // Convert relative path to absolute path + absPath, err := filepath.Abs(path) + if err != nil { + return "", err + } + + // Get the root path of the volume + volumeRoot := filepath.VolumeName(absPath) + "\\" + + volumeRootPtr, _ := syscall.UTF16PtrFromString(volumeRoot) + + var ( + fileSystemNameBuffer = make([]uint16, 260) + nFileSystemNameSize = uint32(len(fileSystemNameBuffer)) + ) + + ret, _, err := syscall.SyscallN(getVolumeInformation, + uintptr(unsafe.Pointer(volumeRootPtr)), + 0, + 0, + 0, + 0, + 0, + uintptr(unsafe.Pointer(&fileSystemNameBuffer[0])), + uintptr(nFileSystemNameSize), + 0) + + if ret == 0 { + return "", err + } + + return syscall.UTF16ToString(fileSystemNameBuffer), nil +} diff --git a/pkg/types/utils.go b/pkg/types/utils.go index e42c36d8a..712d44ba1 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -3,6 +3,7 @@ package types import ( "fmt" "path/filepath" + "strings" "time" log "github.com/sirupsen/logrus" @@ -67,3 +68,12 @@ func ConfigureLogger(clog *log.Logger) error { func UtcNow() time.Time { return time.Now().UTC() } + +func IsNetworkFS(path string) (bool, string, error) { + fsType, err := GetFSType(path) + if err != nil { + return false, "", err + } + fsType = strings.ToLower(fsType) + return fsType == "nfs" || fsType == "cifs" || fsType == "smb" || fsType == "smb2", fsType, nil +}