data provider: add a setting to prevent auto-update

.github/workflows/docker.yml

@@ -66,7 +66,7 @@ jobs:
           build-args: |
             COMMIT_SHA=${{ steps.info.outputs.sha }}
           labels: |
-            org.opencontainers.image.title=${{ github.event.repository.name }}
+            org.opencontainers.image.title=SFTPGo
             org.opencontainers.image.description=Fully featured and highly configurable SFTP server with optional FTP/S and WebDAV support
             org.opencontainers.image.url=${{ github.event.repository.html_url }}
             org.opencontainers.image.source=${{ github.event.repository.clone_url }}

README.md

@@ -90,17 +90,17 @@ sftpgo serve
 
 Check out [this documentation](./docs/service.md) if you want to run SFTPGo as a service.
 
-### Data provider initialization
+### Data provider initialization and update
 
-Before starting the SFTPGo server, please ensure that the configured data provider is properly initialized.
+Before starting the SFTPGo server, please ensure that the configured data provider is properly initialized/updated.
 
-SQL based data providers (SQLite, MySQL, PostgreSQL) require the creation of a database containing the required tables. Memory and bolt data providers do not require an initialization.
+SQL based data providers (SQLite, MySQL, PostgreSQL) require the creation of a database containing the required tables. Memory and bolt data providers do not require an initialization but they could require an update to the existing data after upgrading SFTPGo.
 
 For PostgreSQL and MySQL providers, you need to create the configured database.
 
-SFTPGo will attempt to automatically detect if the data privider has been initialized and if not, initialize it on startup.
+SFTPGo will attempt to automatically detect if the data provider is initialized/updated and if not, will attempt to initialize/ update it on startup as needed.
 
-Alternately, you can create the required data provider structure yourself using the `initprovider` command.
+Alternately, you can create/update the required data provider structures yourself using the `initprovider` command.
 
 For example, you can simply execute the following command from the configuration directory:
 
@@ -114,7 +114,7 @@ Take a look at the CLI usage to learn how to specify a different configuration f
 sftpgo initprovider --help
 ```
 
-After the first initialization (manual or automatic), the database structure will be automatically checked and updated, if required, at startup.
+You can also disable automatic data provider checks at startup setting the `update_mode` configuration key to `1`.
 
 ## Tutorials
 

cmd/initprovider.go

@@ -16,18 +16,20 @@ import (
 var (
 	initProviderCmd = &cobra.Command{
 		Use:   "initprovider",
-		Short: "Initializes the configured data provider",
+		Short: "Initializes and/or updates the configured data provider",
 		Long: `This command reads the data provider connection details from the specified
-configuration file and creates the initial structure.
+configuration file and creates the initial structure or update the existing one,
+as needed.
 
-Some data providers such as bolt and memory does not require an initialization.
+Some data providers such as bolt and memory does not require an initialization
+but they could require an update to the existing data after upgrading SFTPGo.
 
-For SQLite provider the database file will be auto created if missing.
+For SQLite/bolt providers the database file will be auto-created if missing.
 
 For PostgreSQL and MySQL providers you need to create the configured database,
-this command will create the required tables.
+this command will create/update the required tables as needed.
 
-To initialize the data provider from the configuration directory simply use:
+To initialize/update the data provider from the configuration directory simply use:
 
 $ sftpgo initprovider
 
@@ -42,14 +44,14 @@ Please take a look at the usage below to customize the options.`,
 				return
 			}
 			providerConf := config.GetProviderConf()
-			logger.DebugToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed())
+			logger.InfoToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed())
 			err = dataprovider.InitializeDatabase(providerConf, configDir)
 			if err == nil {
-				logger.DebugToConsole("Data provider successfully initialized")
+				logger.InfoToConsole("Data provider successfully initialized/updated")
 			} else if err == dataprovider.ErrNoInitRequired {
-				logger.DebugToConsole("%v", err.Error())
+				logger.InfoToConsole("%v", err.Error())
 			} else {
-				logger.WarnToConsole("Unable to initialize data provider: %v", err)
+				logger.WarnToConsole("Unable to initialize/update the data provider: %v", err)
 				os.Exit(1)
 			}
 		},

config/config.go

@@ -141,6 +141,7 @@ func init() {
 					Parallelism: 2,
 				},
 			},
+			UpdateMode: 0,
 		},
 		HTTPDConfig: httpd.Conf{
 			BindPort:           8080,

dataprovider/bolt.go

@@ -712,8 +712,8 @@ func (p BoltProvider) migrateDatabase() error {
 		return err
 	}
 	if dbVersion.Version == boltDatabaseVersion {
-		providerLog(logger.LevelDebug, "bolt database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "bolt database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -866,6 +866,7 @@ func getFolderBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
 }
 
 func updateDatabaseFrom1To2(dbHandle *bolt.DB) error {
+	logger.InfoToConsole("updating bolt database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating bolt database version: 1 -> 2")
 	usernames, err := getBoltAvailableUsernames(dbHandle)
 	if err != nil {
@@ -887,6 +888,7 @@ func updateDatabaseFrom1To2(dbHandle *bolt.DB) error {
 }
 
 func updateDatabaseFrom2To3(dbHandle *bolt.DB) error {
+	logger.InfoToConsole("updating bolt database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating bolt database version: 2 -> 3")
 	users := []User{}
 	err := dbHandle.View(func(tx *bolt.Tx) error {
@@ -941,6 +943,7 @@ func updateDatabaseFrom2To3(dbHandle *bolt.DB) error {
 }
 
 func updateDatabaseFrom3To4(dbHandle *bolt.DB) error {
+	logger.InfoToConsole("updating bolt database version: 3 -> 4")
 	providerLog(logger.LevelInfo, "updating bolt database version: 3 -> 4")
 	foldersToScan := []string{}
 	users := []userCompactVFolders{}

dataprovider/dataprovider.go

@@ -99,8 +99,8 @@ var (
 	ErrNoAuthTryed = errors.New("no auth tryed")
 	// ValidProtocols defines all the valid protcols
 	ValidProtocols = []string{"SSH", "FTP", "DAV"}
-	// ErrNoInitRequired defines the error returned by InitProvider if no inizialization is required
-	ErrNoInitRequired = errors.New("Data provider initialization is not required")
+	// ErrNoInitRequired defines the error returned by InitProvider if no inizialization/update is required
+	ErrNoInitRequired = errors.New("The data provider is already up to date")
 	// ErrInvalidCredentials defines the error to return if the supplied credentials are invalid
 	ErrInvalidCredentials = errors.New("Invalid credentials")
 	webDAVUsersCache      sync.Map
@@ -251,6 +251,10 @@ type Config struct {
 	CheckPasswordScope int `json:"check_password_scope" mapstructure:"check_password_scope"`
 	// PasswordHashing defines the configuration for password hashing
 	PasswordHashing PasswordHashing `json:"password_hashing" mapstructure:"password_hashing"`
+	// Defines how the database will be initialized/updated:
+	// - 0 means automatically
+	// - 1 means manually using the initprovider sub-command
+	UpdateMode int `json:"update_mode" mapstructure:"update_mode"`
 }
 
 // BackupData defines the structure for the backup/restore files
@@ -383,19 +387,23 @@ func Initialize(cnf Config, basePath string) error {
 	if err != nil {
 		return err
 	}
-	err = provider.initializeDatabase()
-	if err != nil && err != ErrNoInitRequired {
-		logger.WarnToConsole("Unable to initialize data provider: %v", err)
-		providerLog(logger.LevelWarn, "Unable to initialize data provider: %v", err)
-		return err
-	}
-	if err == nil {
-		logger.DebugToConsole("Data provider successfully initialized")
-	}
-	err = provider.migrateDatabase()
-	if err != nil {
-		providerLog(logger.LevelWarn, "database migration error: %v", err)
-		return err
+	if cnf.UpdateMode == 0 {
+		err = provider.initializeDatabase()
+		if err != nil && err != ErrNoInitRequired {
+			logger.WarnToConsole("Unable to initialize data provider: %v", err)
+			providerLog(logger.LevelWarn, "Unable to initialize data provider: %v", err)
+			return err
+		}
+		if err == nil {
+			logger.DebugToConsole("Data provider successfully initialized")
+		}
+		err = provider.migrateDatabase()
+		if err != nil && err != ErrNoInitRequired {
+			providerLog(logger.LevelWarn, "database migration error: %v", err)
+			return err
+		}
+	} else {
+		providerLog(logger.LevelInfo, "database initialization/migration skipped, manual mode is configured")
 	}
 	argon2Params = &argon2id.Params{
 		Memory:      cnf.PasswordHashing.Argon2Options.Memory,
@@ -458,14 +466,15 @@ func validateSQLTablesPrefix() error {
 func InitializeDatabase(cnf Config, basePath string) error {
 	config = cnf
 
-	if config.Driver == BoltDataProviderName || config.Driver == MemoryDataProviderName {
-		return ErrNoInitRequired
-	}
 	err := createProvider(basePath)
 	if err != nil {
 		return err
 	}
-	return provider.initializeDatabase()
+	err = provider.initializeDatabase()
+	if err != nil && err != ErrNoInitRequired {
+		return err
+	}
+	return provider.migrateDatabase()
 }
 
 // CheckUserAndPass retrieves the SFTP user with the given username and password if a match is found or an error

dataprovider/memory.go

@@ -671,5 +671,5 @@ func (p MemoryProvider) initializeDatabase() error {
 }
 
 func (p MemoryProvider) migrateDatabase() error {
-	return nil
+	return ErrNoInitRequired
 }

dataprovider/mysql.go

@@ -205,8 +205,8 @@ func (p MySQLProvider) migrateDatabase() error {
 		return err
 	}
 	if dbVersion.Version == sqlDatabaseVersion {
-		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -233,12 +233,14 @@ func (p MySQLProvider) migrateDatabase() error {
 }
 
 func updateMySQLDatabaseFrom1To2(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
 	sql := strings.Replace(mysqlV2SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
 }
 
 func updateMySQLDatabaseFrom2To3(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
 	sql := strings.Replace(mysqlV3SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)

dataprovider/pgsql.go

@@ -204,8 +204,8 @@ func (p PGSQLProvider) migrateDatabase() error {
 		return err
 	}
 	if dbVersion.Version == sqlDatabaseVersion {
-		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -232,12 +232,14 @@ func (p PGSQLProvider) migrateDatabase() error {
 }
 
 func updatePGSQLDatabaseFrom1To2(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
 	sql := strings.Replace(pgsqlV2SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
 }
 
 func updatePGSQLDatabaseFrom2To3(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
 	sql := strings.Replace(pgsqlV3SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)

dataprovider/sqlcommon.go

@@ -899,6 +899,7 @@ func sqlCommonRestoreCompatVirtualFolders(ctx context.Context, users []userCompa
 }
 
 func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 3 -> 4")
 	providerLog(logger.LevelInfo, "updating database version: 3 -> 4")
 	users, err := sqlCommonGetCompatVirtualFolders(dbHandle)
 	if err != nil {

dataprovider/sqlite.go

@@ -227,8 +227,8 @@ func (p SQLiteProvider) migrateDatabase() error {
 		return err
 	}
 	if dbVersion.Version == sqlDatabaseVersion {
-		providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version)
-		return nil
+		providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version)
+		return ErrNoInitRequired
 	}
 	switch dbVersion.Version {
 	case 1:
@@ -255,12 +255,14 @@ func (p SQLiteProvider) migrateDatabase() error {
 }
 
 func updateSQLiteDatabaseFrom1To2(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 1 -> 2")
 	providerLog(logger.LevelInfo, "updating database version: 1 -> 2")
 	sql := strings.Replace(sqliteV2SQL, "{{users}}", sqlTableUsers, 1)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2)
 }
 
 func updateSQLiteDatabaseFrom2To3(dbHandle *sql.DB) error {
+	logger.InfoToConsole("updating database version: 2 -> 3")
 	providerLog(logger.LevelInfo, "updating database version: 2 -> 3")
 	sql := strings.ReplaceAll(sqliteV3SQL, "{{users}}", sqlTableUsers)
 	return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3)

docs/full-configuration.md

@@ -11,7 +11,7 @@ Usage:
 Available Commands:
   gen          A collection of useful generators
   help         Help about any command
-  initprovider Initializes the configured data provider
+  initprovider Initializes and/or updates the configured data provider
   portable     Serve a single directory
   serve        Start the SFTP Server
 
@@ -143,6 +143,7 @@ The configuration file contains the following sections:
       - `memory`, unsigned integer. The amount of memory used by the algorithm (in kibibytes). Default: 65536.
       - `iterations`, unsigned integer. The number of iterations over the memory. Default: 1.
       - `parallelism`. unsigned 8 bit integer. The number of threads (or lanes) used by the algorithm. Default: 2.
+  - `update_mode`, integer. Defines how the database will be initialized/updated. 0 means automatically. 1 means manually using the initprovider sub-command.
 - **"httpd"**, the configuration for the HTTP server used to serve REST API and to expose the built-in web interface
   - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080
   - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"

sftpgo.json

@@ -99,7 +99,8 @@
         "iterations": 1,
         "parallelism": 2
       }
-    }
+    },
+    "update_mode": 0
   },
   "httpd": {
     "bind_port": 8080,