Allow to read env vars from files inside the "env.d" directory

This makes it easier to set environment variables on some operating systems.
Setting configuration options from environment variables is recommended if
you want to avoid the time-consuming task of merging your changes with the
default configuration file after upgrading SFTPGo

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

docs/full-configuration.md

@@ -465,6 +465,12 @@ You can select `sha256-simd` setting the environment variable `SFTPGO_MINIO_SHA2
 
  `sha256-simd` is particularly useful if you have an Intel CPU with SHA extensions or an ARM CPU with Cryptography Extensions.
 
+The SFTPGo configuration file can change between different versions and merging your custom settings with the default config file may be time-consuming. For this reason we suggest to set your custom settings using environment variables. This eliminates the need to merge your changes with the default configuration file after each update, you have to just check that your configuration key still exists.
+Setting configuration options from environment variables is natural in Docker/Kubernetes.
+If you install SFTPGo on Linux using the official deb/rpm packages you can set your custom environment variables in the file `/etc/sftpgo/sftpgo.env` (create this file if it does not exist).
+SFTPGo also reads files inside the `env.d` directory relative to config dir and then export the valid variables into environment variables if they are not already set. With this method you can override any configuration options, set environment variables for SFTPGo plugins but you cannot set command flags because these files are read after that SFTPGo starts and the config dir must already be set.
+Of course you can also set environment variables with the method provided by the operating system of your choice.
+
 </details>
 
 <details><summary><font size=5>Binding to privileged ports</font></summary>

go.mod

@@ -58,6 +58,7 @@ require (
 	github.com/spf13/viper v1.13.0
 	github.com/stretchr/testify v1.8.0
 	github.com/studio-b12/gowebdav v0.0.0-20221012160928-e70a598e946e
+	github.com/subosito/gotenv v1.4.1
 	github.com/unrolled/secure v1.13.0
 	github.com/wagslane/go-password-validator v0.3.0
 	github.com/xhit/go-simple-mail/v2 v2.12.0
@@ -148,7 +149,6 @@ require (
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.5.0 // indirect
 	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect

internal/config/config.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"github.com/spf13/viper"
+	"github.com/subosito/gotenv"
 
 	"github.com/drakkan/sftpgo/v2/internal/acme"
 	"github.com/drakkan/sftpgo/v2/internal/command"
@@ -634,6 +635,30 @@ func setConfigFile(configDir, configFile string) {
 	viper.SetConfigFile(configFile)
 }
 
+// readEnvFiles reads files inside the "env.d" directory relative to configDir
+// and then export the valid variables into environment variables if they do
+// not exist
+func readEnvFiles(configDir string) {
+	envd := filepath.Join(configDir, "env.d")
+	entries, err := os.ReadDir(envd)
+	if err != nil {
+		logger.Info(logSender, "", "unable to read env files from %q: %v", envd, err)
+		return
+	}
+	for _, entry := range entries {
+		info, err := entry.Info()
+		if err == nil && info.Mode().IsRegular() {
+			envFile := filepath.Join(envd, entry.Name())
+			err = gotenv.Load(envFile)
+			if err != nil {
+				logger.Error(logSender, "", "unable to load env vars from file %q, err: %v", envFile, err)
+			} else {
+				logger.Info(logSender, "", "set env vars from file %q", envFile)
+			}
+		}
+	}
+}
+
 // LoadConfig loads the configuration
 // configDir will be added to the configuration search paths.
 // The search path contains by default the current directory and on linux it contains
@@ -641,6 +666,7 @@ func setConfigFile(configDir, configFile string) {
 // configFile is an absolute or relative path (to the config dir) to the configuration file.
 func LoadConfig(configDir, configFile string) error {
 	var err error
+	readEnvFiles(configDir)
 	viper.AddConfigPath(configDir)
 	setViperAdditionalConfigPaths()
 	viper.AddConfigPath(".")

internal/config/config_test.go

@@ -89,6 +89,29 @@ func TestLoadConfigFileNotFound(t *testing.T) {
 	assert.Len(t, mfaConf.TOTP, 1)
 }
 
+func TestReadEnvFiles(t *testing.T) {
+	reset()
+
+	envd := filepath.Join(configDir, "env.d")
+	err := os.Mkdir(envd, os.ModePerm)
+	assert.NoError(t, err)
+
+	err = os.WriteFile(filepath.Join(envd, "env1"), []byte("SFTPGO_SFTPD__MAX_AUTH_TRIES = 10"), 0666)
+	assert.NoError(t, err)
+	err = os.WriteFile(filepath.Join(envd, "env2"), []byte(`{"invalid env": "value"}`), 0666)
+	assert.NoError(t, err)
+
+	err = config.LoadConfig(configDir, "")
+	assert.NoError(t, err)
+	assert.Equal(t, 10, config.GetSFTPDConfig().MaxAuthTries)
+
+	_, ok := os.LookupEnv("SFTPGO_SFTPD__MAX_AUTH_TRIES")
+	assert.True(t, ok)
+	err = os.Unsetenv("SFTPGO_SFTPD__MAX_AUTH_TRIES")
+	assert.NoError(t, err)
+	os.RemoveAll(envd)
+}
+
 func TestEmptyBanner(t *testing.T) {
 	reset()
 

pkgs/build.sh

@@ -93,6 +93,9 @@ contents:
   - dst: "/var/lib/sftpgo"
     type: dir
 
+  - dst: "/etc/sftpgo/env.d"
+    type: dir
+
 overrides:
   deb:
     recommends:

pkgs/choco/tools/ChocolateyInstall.ps1

@@ -22,6 +22,7 @@ Install-ChocolateyPackage @packageArgs
 
 $DefaultDataPath = Join-Path -Path $ENV:ProgramData -ChildPath "SFTPGo"
 $DefaultConfigurationFilePath = Join-Path -Path $DefaultDataPath -ChildPath "sftpgo.json"
+$EnvDirPath = Join-Path -Path $DefaultDataPath -ChildPath "env.d"
 
 # `t = tab
 Write-Output "---------------------------"
@@ -38,7 +39,8 @@ Write-Output "Default data location:"
 Write-Output "`t$DefaultDataPath"
 Write-Output "Default configuration file location:"
 Write-Output "`t$DefaultConfigurationFilePath"
-Write-Output ""
+Write-Output "Directory to create environment variable files to set configuration options:"
+Write-Output "`t$EnvDirPath"
 Write-Output "If the SFTPGo service does not start, make sure that TCP ports 2022 and 8080 are"
 Write-Output "not used by other services or change the SFTPGo configuration to suit your needs."
 Write-Output ""

pkgs/debian/postinst

@@ -24,15 +24,15 @@ if [ "$1" = "configure" ]; then
     sftpgo initprovider -c /etc/sftpgo
     # ensure files and folders have the appropriate permissions
     chown -R sftpgo:sftpgo /etc/sftpgo /var/lib/sftpgo /srv/sftpgo
-    chmod 750 /etc/sftpgo /var/lib/sftpgo /srv/sftpgo
+    chmod 750 /etc/sftpgo /etc/sftpgo/env.d /var/lib/sftpgo /srv/sftpgo
     chmod 640 /etc/sftpgo/sftpgo.json
   fi
 
-  # we added /srv/sftpgo after 1.1.0, we should check if we are upgrading
+  # we added /etc/sftpgo/env.d in v2.4.0, we should check if we are upgrading
-  # from this version but a non-recursive chmod/chown shouldn't hurt
+  # from a previous version but a non-recursive chmod/chown shouldn't hurt
-  if [ -d /srv/sftpgo ]; then
+  if [ -d /etc/sftpgo/env.d ]; then
-    chown sftpgo:sftpgo /srv/sftpgo
+    chown sftpgo:sftpgo /etc/sftpgo/env.d
-    chmod 750 /srv/sftpgo
+    chmod 750 /etc/sftpgo/env.d
   fi
 
   # set the cap_net_bind_service capability so the service can bind to privileged ports

pkgs/debian/sftpgo.dirs

@@ -1,2 +1,3 @@
 /var/lib/sftpgo
 /srv/sftpgo
+/etc/sftpgo/env.d

pkgs/scripts/deb/postinstall.sh

@@ -24,15 +24,15 @@ if [ "$1" = "configure" ]; then
     sftpgo initprovider -c /etc/sftpgo
     # ensure files and folders have the appropriate permissions
     chown -R sftpgo:sftpgo /etc/sftpgo /var/lib/sftpgo /srv/sftpgo
-    chmod 750 /etc/sftpgo /var/lib/sftpgo /srv/sftpgo
+    chmod 750 /etc/sftpgo /etc/sftpgo/env.d /var/lib/sftpgo /srv/sftpgo
     chmod 640 /etc/sftpgo/sftpgo.json
   fi
 
-  # we added /srv/sftpgo after 1.1.0, we should check if we are upgrading
+  # we added /etc/sftpgo/env.d in v2.4.0, we should check if we are upgrading
-  # from this version but a non-recursive chmod/chown shouldn't hurt
+  # from a previous version but a non-recursive chmod/chown shouldn't hurt
-  if [ -d /srv/sftpgo ]; then
+  if [ -d /etc/sftpgo/env.d ]; then
-    chown sftpgo:sftpgo /srv/sftpgo
+    chown sftpgo:sftpgo /etc/sftpgo/env.d
-    chmod 750 /srv/sftpgo
+    chmod 750 /etc/sftpgo/env.d
   fi
 
   # set the cap_net_bind_service capability so the service can bind to privileged ports

pkgs/scripts/rpm/postinstall

@@ -17,16 +17,21 @@ if [ $1 -eq 1 ]; then
   /usr/bin/sftpgo initprovider -c /etc/sftpgo
   # ensure files and folders have the appropriate permissions
   /usr/bin/chown -R sftpgo:sftpgo /etc/sftpgo /var/lib/sftpgo /srv/sftpgo
-  /usr/bin/chmod 750 /etc/sftpgo /var/lib/sftpgo /srv/sftpgo
+  /usr/bin/chmod 750 /etc/sftpgo /etc/sftpgo/env.d /var/lib/sftpgo /srv/sftpgo
   /usr/bin/chmod 640 /etc/sftpgo/sftpgo.json
 fi
 
-# adjust permissions for /srv/sftpgo and /var/lib/sftpgo
+# adjust permissions for /srv/sftpgo, /etc/sftpgo/env.d and /var/lib/sftpgo
 if [ -d /srv/sftpgo ]; then
     /usr/bin/chown sftpgo:sftpgo /srv/sftpgo
     /usr/bin/chmod 750 /srv/sftpgo
 fi
 
+if [ -d /etc/sftpgo/env.d ]; then
+    /usr/bin/chown sftpgo:sftpgo /etc/sftpgo/env.d
+    /usr/bin/chmod 750 /etc/sftpgo/env.d
+fi
+
 if [ -d /var/lib/sftpgo ]; then
     /usr/bin/chown sftpgo:sftpgo /var/lib/sftpgo
     /usr/bin/chmod 750 /var/lib/sftpgo

windows-installer/sftpgo.iss

@@ -73,7 +73,7 @@ Source: "README.txt"; DestDir: "{app}"; Flags: ignoreversion isreadme
 [Dirs]
 Name: "{commonappdata}\{#MyAppName}\logs"; Permissions: everyone-full
 Name: "{commonappdata}\{#MyAppName}\backups"; Permissions: everyone-full
-Name: "{commonappdata}\{#MyAppName}\credentials"; Permissions: everyone-full
+Name: "{commonappdata}\{#MyAppName}\env.d"; Permissions: everyone-full
 Name: "{commonappdata}\{#MyAppName}\certs"; Permissions: everyone-full
 
 [Icons]