Auto detect if reading logs or storing sqlite db on a network share (#2241)

pkg/acquisition/modules/file/file.go

@@ -40,7 +40,7 @@ type FileConfiguration struct {
 	Filename                          string
 	ForceInotify                      bool  `yaml:"force_inotify"`
 	MaxBufferSize                     int   `yaml:"max_buffer_size"`
-	PollWithoutInotify                bool `yaml:"poll_without_inotify"`
+	PollWithoutInotify                *bool `yaml:"poll_without_inotify"`
 	configuration.DataSourceCommonCfg `yaml:",inline"`
 }
 
@@ -330,7 +330,22 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er
 			continue
 		}
 
-		tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())})
+		inotifyPoll := true
+		if f.config.PollWithoutInotify != nil {
+			inotifyPoll = *f.config.PollWithoutInotify
+		} else {
+			networkFS, fsType, err := types.IsNetworkFS(file)
+			if err != nil {
+				f.logger.Warningf("Could not get fs type for %s : %s", file, err)
+			}
+			f.logger.Debugf("fs for %s is network: %t (%s)", file, networkFS, fsType)
+			if networkFS {
+				f.logger.Warnf("Disabling inotify poll on %s as it is on a network share. You can manually set poll_without_inotify to true to make this message disappear, or to false to enforce inotify poll", file)
+				inotifyPoll = false
+			}
+		}
+
+		tail, err := tail.TailFile(file, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekEnd}, Logger: log.NewEntry(log.StandardLogger())})
 		if err != nil {
 			f.logger.Errorf("Could not start tailing file %s : %s", file, err)
 			continue
@@ -413,8 +428,27 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error {
 					f.logger.Errorf("unable to close %s : %s", event.Name, err)
 					continue
 				}
+
+				inotifyPoll := true
+				if f.config.PollWithoutInotify != nil {
+					inotifyPoll = *f.config.PollWithoutInotify
+				} else {
+					if f.config.PollWithoutInotify != nil {
+						inotifyPoll = *f.config.PollWithoutInotify
+					} else {
+						networkFS, fsType, err := types.IsNetworkFS(event.Name)
+						if err != nil {
+							f.logger.Warningf("Could not get fs type for %s : %s", event.Name, err)
+						}
+						f.logger.Debugf("fs for %s is network: %t (%s)", event.Name, networkFS, fsType)
+						if networkFS {
+							inotifyPoll = false
+						}
+					}
+				}
+
 				//Slightly different parameters for Location, as we want to read the first lines of the newly created file
-				tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: f.config.PollWithoutInotify, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}})
+				tail, err := tail.TailFile(event.Name, tail.Config{ReOpen: true, Follow: true, Poll: inotifyPoll, Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}})
 				if err != nil {
 					logger.Errorf("Could not start tailing file %s : %s", event.Name, err)
 					break

pkg/csconfig/api_test.go

@@ -194,6 +194,7 @@ func TestLoadAPIServer(t *testing.T) {
 					DbPath:           "./testdata/test.db",
 					Type:             "sqlite",
 					MaxOpenConns:     ptr.Of(DEFAULT_MAX_OPEN_CONNS),
+					UseWal:           ptr.Of(true), // autodetected
 					DecisionBulkSize: defaultDecisionBulkSize,
 				},
 				ConsoleConfigPath: DefaultConfigPath("console.yaml"),

pkg/csconfig/database.go

@@ -3,12 +3,15 @@ package csconfig
 import (
 	"errors"
 	"fmt"
+	"path/filepath"
 	"time"
 
 	"entgo.io/ent/dialect"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/crowdsecurity/go-cs-lib/ptr"
+
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
 const (
@@ -69,6 +72,35 @@ func (c *Config) LoadDBConfig(inCli bool) error {
 		c.DbConfig.MaxOpenConns = ptr.Of(DEFAULT_MAX_OPEN_CONNS)
 	}
 
+	if !inCli && c.DbConfig.Type == "sqlite" {
+		if c.DbConfig.UseWal == nil {
+			dbDir := filepath.Dir(c.DbConfig.DbPath)
+			isNetwork, fsType, err := types.IsNetworkFS(dbDir)
+			if err != nil {
+				log.Warnf("unable to determine if database is on network filesystem: %s", err)
+				log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.")
+				return nil
+			}
+			if isNetwork {
+				log.Debugf("database is on network filesystem (%s), setting useWal to false", fsType)
+				c.DbConfig.UseWal = ptr.Of(false)
+			} else {
+				log.Debugf("database is on local filesystem (%s), setting useWal to true", fsType)
+				c.DbConfig.UseWal = ptr.Of(true)
+			}
+		} else if *c.DbConfig.UseWal {
+			dbDir := filepath.Dir(c.DbConfig.DbPath)
+			isNetwork, fsType, err := types.IsNetworkFS(dbDir)
+			if err != nil {
+				log.Warnf("unable to determine if database is on network filesystem: %s", err)
+				return nil
+			}
+			if isNetwork {
+				log.Warnf("database seems to be stored on a network share (%s), but useWal is set to true. Proceed at your own risk.", fsType)
+			}
+		}
+	}
+
 	if c.DbConfig.DecisionBulkSize == 0 {
 		log.Tracef("No decision_bulk_size value provided, using default value of %d", defaultDecisionBulkSize)
 		c.DbConfig.DecisionBulkSize = defaultDecisionBulkSize
@@ -79,10 +111,6 @@ func (c *Config) LoadDBConfig(inCli bool) error {
 		c.DbConfig.DecisionBulkSize = maxDecisionBulkSize
 	}
 
-	if !inCli && c.DbConfig.Type == "sqlite" && c.DbConfig.UseWal == nil {
-		log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.")
-	}
-
 	return nil
 }
 

pkg/csconfig/database_test.go

@@ -33,6 +33,7 @@ func TestLoadDBConfig(t *testing.T) {
 				Type:         "sqlite",
 				DbPath:       "./testdata/test.db",
 				MaxOpenConns: ptr.Of(10),
+				UseWal:       ptr.Of(true),
 				DecisionBulkSize: defaultDecisionBulkSize,
 			},
 		},

pkg/types/getfstype.go

@@ -0,0 +1,112 @@
+//go:build !windows
+
+package types
+
+import (
+	"fmt"
+	"syscall"
+)
+
+// Generated with `man statfs | grep _MAGIC | awk '{split(tolower($1),a,"_"); print $2 ": \"" a[1] "\","}'`
+// ext2/3/4 duplicates removed to just have ext4
+// XIAFS removed as well
+var fsTypeMapping map[int]string = map[int]string{
+	0xadf5:     "adfs",
+	0xadff:     "affs",
+	0x5346414f: "afs",
+	0x09041934: "anon",
+	0x0187:     "autofs",
+	0x62646576: "bdevfs",
+	0x42465331: "befs",
+	0x1badface: "bfs",
+	0x42494e4d: "binfmtfs",
+	0xcafe4a11: "bpf",
+	0x9123683e: "btrfs",
+	0x73727279: "btrfs",
+	0x27e0eb:   "cgroup",
+	0x63677270: "cgroup2",
+	0xff534d42: "cifs",
+	0x73757245: "coda",
+	0x012ff7b7: "coh",
+	0x28cd3d45: "cramfs",
+	0x64626720: "debugfs",
+	0x1373:     "devfs",
+	0x1cd1:     "devpts",
+	0xf15f:     "ecryptfs",
+	0xde5e81e4: "efivarfs",
+	0x00414a53: "efs",
+	0x137d:     "ext",
+	0xef51:     "ext2",
+	0xef53:     "ext4",
+	0xf2f52010: "f2fs",
+	0x65735546: "fuse",
+	0xbad1dea:  "futexfs",
+	0x4244:     "hfs",
+	0x00c0ffee: "hostfs",
+	0xf995e849: "hpfs",
+	0x958458f6: "hugetlbfs",
+	0x9660:     "isofs",
+	0x72b6:     "jffs2",
+	0x3153464a: "jfs",
+	0x137f:     "minix",
+	0x138f:     "minix",
+	0x2468:     "minix2",
+	0x2478:     "minix2",
+	0x4d5a:     "minix3",
+	0x19800202: "mqueue",
+	0x4d44:     "msdos",
+	0x11307854: "mtd",
+	0x564c:     "ncp",
+	0x6969:     "nfs",
+	0x3434:     "nilfs",
+	0x6e736673: "nsfs",
+	0x5346544e: "ntfs",
+	0x7461636f: "ocfs2",
+	0x9fa1:     "openprom",
+	0x794c7630: "overlayfs",
+	0x50495045: "pipefs",
+	0x9fa0:     "proc",
+	0x6165676c: "pstorefs",
+	0x002f:     "qnx4",
+	0x68191122: "qnx6",
+	0x858458f6: "ramfs",
+	0x52654973: "reiserfs",
+	0x7275:     "romfs",
+	0x73636673: "securityfs",
+	0xf97cff8c: "selinux",
+	0x43415d53: "smack",
+	0x517b:     "smb",
+	0xfe534d42: "smb2",
+	0x534f434b: "sockfs",
+	0x73717368: "squashfs",
+	0x62656572: "sysfs",
+	0x012ff7b6: "sysv2",
+	0x012ff7b5: "sysv4",
+	0x01021994: "tmpfs",
+	0x74726163: "tracefs",
+	0x15013346: "udf",
+	0x00011954: "ufs",
+	0x9fa2:     "usbdevice",
+	0x01021997: "v9fs",
+	0xa501fcf5: "vxfs",
+	0xabba1974: "xenfs",
+	0x012ff7b4: "xenix",
+	0x58465342: "xfs",
+}
+
+func GetFSType(path string) (string, error) {
+	var buf syscall.Statfs_t
+
+	err := syscall.Statfs(path, &buf)
+
+	if err != nil {
+		return "", err
+	}
+
+	fsType, ok := fsTypeMapping[int(buf.Type)]
+	if !ok {
+		return "", fmt.Errorf("unknown fstype %d", buf.Type)
+	}
+
+	return fsType, nil
+}

pkg/types/getfstype_windows.go

@@ -0,0 +1,53 @@
+package types
+
+import (
+	"path/filepath"
+	"syscall"
+	"unsafe"
+)
+
+func GetFSType(path string) (string, error) {
+	kernel32, err := syscall.LoadLibrary("kernel32.dll")
+	if err != nil {
+		return "", err
+	}
+	defer syscall.FreeLibrary(kernel32)
+
+	getVolumeInformation, err := syscall.GetProcAddress(kernel32, "GetVolumeInformationW")
+	if err != nil {
+		return "", err
+	}
+
+	// Convert relative path to absolute path
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return "", err
+	}
+
+	// Get the root path of the volume
+	volumeRoot := filepath.VolumeName(absPath) + "\\"
+
+	volumeRootPtr, _ := syscall.UTF16PtrFromString(volumeRoot)
+
+	var (
+		fileSystemNameBuffer = make([]uint16, 260)
+		nFileSystemNameSize  = uint32(len(fileSystemNameBuffer))
+	)
+
+	ret, _, err := syscall.SyscallN(getVolumeInformation,
+		uintptr(unsafe.Pointer(volumeRootPtr)),
+		0,
+		0,
+		0,
+		0,
+		0,
+		uintptr(unsafe.Pointer(&fileSystemNameBuffer[0])),
+		uintptr(nFileSystemNameSize),
+		0)
+
+	if ret == 0 {
+		return "", err
+	}
+
+	return syscall.UTF16ToString(fileSystemNameBuffer), nil
+}

pkg/types/utils.go

@@ -3,6 +3,7 @@ package types
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -67,3 +68,12 @@ func ConfigureLogger(clog *log.Logger) error {
 func UtcNow() time.Time {
 	return time.Now().UTC()
 }
+
+func IsNetworkFS(path string) (bool, string, error) {
+	fsType, err := GetFSType(path)
+	if err != nil {
+		return false, "", err
+	}
+	fsType = strings.ToLower(fsType)
+	return fsType == "nfs" || fsType == "cifs" || fsType == "smb" || fsType == "smb2", fsType, nil
+}