beenull/sftpgo-mirror
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2024-11-21 23:20:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

squash database migrations

Browse source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino 2024-06-15 16:02:09 +02:00
parent bd5b32101f
commit c5c42f072b
No known key found for this signature in database
GPG key ID: 935D2952DEC4EECF
6 changed files with 29 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
internal/cmd/revertprovider.go
View file

@ -40,8 +40,8 @@ Please take a look at the usage below to customize the options.`,
Run: func(_ *cobra.Command, _ []string) {
logger.DisableLogger()
logger.EnableConsoleLogger(zerolog.DebugLevel)
if revertProviderTargetVersion != 28 {
logger.WarnToConsole("Unsupported target version, 28 is the only supported one")
if revertProviderTargetVersion != 29 {
logger.WarnToConsole("Unsupported target version, 29 is the only supported one")
os.Exit(1)
}
configDir = util.CleanDirInput(configDir)
@ -71,7 +71,7 @@ Please take a look at the usage below to customize the options.`,
func init() {
addConfigFlags(revertProviderCmd)
revertProviderCmd.Flags().IntVar(&revertProviderTargetVersion, "to-version", 28, `28 means the version supported in v2.5.x`)
revertProviderCmd.Flags().IntVar(&revertProviderTargetVersion, "to-version", 29, `29 means the version supported in v2.6.x`)
rootCmd.AddCommand(revertProviderCmd)
}

16
internal/dataprovider/bolt.go
View file

@ -3134,15 +3134,11 @@ func (p *BoltProvider) migrateDatabase() error {
case version == boltDatabaseVersion:
providerLog(logger.LevelDebug, "bolt database is up to date, current version: %d", version)
return ErrNoInitRequired
case version < 28:
case version < 29:
err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err)
return err
case version == 28:
logger.InfoToConsole("updating database schema version: %d -> 29", version)
providerLog(logger.LevelInfo, "updating database schema version: %d -> 29", version)
return updateBoltDatabaseVersion(p.dbHandle, 29)
default:
if version > boltDatabaseVersion {
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
@ -3164,10 +3160,6 @@ func (p *BoltProvider) revertDatabase(targetVersion int) error { //nolint:gocycl
return errors.New("current version match target version, nothing to do")
}
switch dbVersion.Version {
case 29:
logger.InfoToConsole("downgrading database schema version: %d -> 28", dbVersion.Version)
providerLog(logger.LevelInfo, "downgrading database schema version: %d -> 28", dbVersion.Version)
return updateBoltDatabaseVersion(p.dbHandle, 28)
default:
return fmt.Errorf("database schema version not handled: %v", dbVersion.Version)
}
@ -3899,7 +3891,7 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) {
v := bucket.Get(dbVersionKey)
if v == nil {
dbVersion = schemaVersion{
Version: 28,
Version: 29,
}
return nil
}
@ -3908,7 +3900,7 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) {
return dbVersion, err
}
func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error {
/*func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error {
err := dbHandle.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(dbVersionBucket)
if bucket == nil {
@ -3924,4 +3916,4 @@ func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error {
return bucket.Put(dbVersionKey, buf)
})
return err
}
}*/

7
internal/dataprovider/dataprovider.go
View file

@ -929,12 +929,13 @@ func checkDatabase(checkAdmins bool) error {
if config.UpdateMode == 0 {
err := provider.initializeDatabase()
if err != nil && err != ErrNoInitRequired {
logger.WarnToConsole("Unable to initialize data provider: %v", err)
providerLog(logger.LevelError, "Unable to initialize data provider: %v", err)
logger.WarnToConsole("unable to initialize data provider: %v", err)
providerLog(logger.LevelError, "unable to initialize data provider: %v", err)
return err
}
if err == nil {
logger.DebugToConsole("Data provider successfully initialized")
logger.DebugToConsole("data provider successfully initialized")
providerLog(logger.LevelInfo, "data provider successfully initialized")
}
err = provider.migrateDatabase()
if err != nil && err != ErrNoInitRequired {

38
internal/dataprovider/mysql.go
View file

@ -95,8 +95,8 @@ const (
"`last_login` bigint NOT NULL, `filters` longtext NULL, `filesystem` longtext NULL, `additional_info` longtext NULL, " +
"`created_at` bigint NOT NULL, `updated_at` bigint NOT NULL, `email` varchar(255) NULL, " +
"`upload_data_transfer` integer NOT NULL, `download_data_transfer` integer NOT NULL, " +
"`total_data_transfer` integer NOT NULL, `used_upload_data_transfer` integer NOT NULL, " +
"`used_download_data_transfer` integer NOT NULL, `deleted_at` bigint NOT NULL, `first_download` bigint NOT NULL, " +
"`total_data_transfer` integer NOT NULL, `used_upload_data_transfer` bigint NOT NULL, " +
"`used_download_data_transfer` bigint NOT NULL, `deleted_at` bigint NOT NULL, `first_download` bigint NOT NULL, " +
"`first_upload` bigint NOT NULL, `last_password_change` bigint NOT NULL, `role_id` integer NULL);" +
"CREATE TABLE `{{groups_folders_mapping}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, " +
"`group_id` integer NOT NULL, `folder_id` integer NOT NULL, " +
@ -193,11 +193,7 @@ const (
"CREATE INDEX `{{prefix}}ip_lists_updated_at_idx` ON `{{ip_lists}}` (`updated_at`);" +
"CREATE INDEX `{{prefix}}ip_lists_deleted_at_idx` ON `{{ip_lists}}` (`deleted_at`);" +
"CREATE INDEX `{{prefix}}ip_lists_first_last_idx` ON `{{ip_lists}}` (`first`, `last`);" +
"INSERT INTO {{schema_version}} (version) VALUES (28);"
mysqlV29SQL = "ALTER TABLE `{{users}}` MODIFY `used_download_data_transfer` bigint NOT NULL;" +
"ALTER TABLE `{{users}}` MODIFY `used_upload_data_transfer` bigint NOT NULL;"
mysqlV29DownSQL = "ALTER TABLE `{{users}}` MODIFY `used_upload_data_transfer` integer NOT NULL;" +
"ALTER TABLE `{{users}}` MODIFY `used_download_data_transfer` integer NOT NULL;"
"INSERT INTO {{schema_version}} (version) VALUES (29);"
)
// MySQLProvider defines the auth provider for MySQL/MariaDB database
@ -776,11 +772,11 @@ func (p *MySQLProvider) initializeDatabase() error {
if errors.Is(err, sql.ErrNoRows) {
return errSchemaVersionEmpty
}
logger.InfoToConsole("creating initial database schema, version 28")
providerLog(logger.LevelInfo, "creating initial database schema, version 28")
logger.InfoToConsole("creating initial database schema, version 29")
providerLog(logger.LevelInfo, "creating initial database schema, version 29")
initialSQL := sqlReplaceAll(mysqlInitialSQL)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, strings.Split(initialSQL, ";"), 28, true)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, strings.Split(initialSQL, ";"), 29, true)
}
func (p *MySQLProvider) migrateDatabase() error {
@ -793,13 +789,11 @@ func (p *MySQLProvider) migrateDatabase() error {
case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version)
return ErrNoInitRequired
case version < 28:
case version < 29:
err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err)
return err
case version == 28:
return updateMySQLDatabaseFrom28To29(p.dbHandle)
default:
if version > sqlDatabaseVersion {
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
@ -822,8 +816,6 @@ func (p *MySQLProvider) revertDatabase(targetVersion int) error {
}
switch dbVersion.Version {
case 29:
return downgradeMySQLDatabaseFrom29To28(p.dbHandle)
default:
return fmt.Errorf("database schema version not handled: %d", dbVersion.Version)
}
@ -861,19 +853,3 @@ func (p *MySQLProvider) normalizeError(err error, fieldType int) error {
}
return err
}
func updateMySQLDatabaseFrom28To29(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 28 -> 29")
providerLog(logger.LevelInfo, "updating database schema version: 28 -> 29")
sql := strings.ReplaceAll(mysqlV29SQL, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 29, true)
}
func downgradeMySQLDatabaseFrom29To28(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 29 -> 28")
providerLog(logger.LevelInfo, "downgrading database schema version: 29 -> 28")
sql := strings.ReplaceAll(mysqlV29DownSQL, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 28, false)
}

38
internal/dataprovider/pgsql.go
View file

@ -95,7 +95,7 @@ CREATE TABLE "{{users}}" ("id" integer NOT NULL PRIMARY KEY GENERATED ALWAYS AS
"download_bandwidth" integer NOT NULL, "last_login" bigint NOT NULL, "filters" text NULL, "filesystem" text NULL,
"additional_info" text NULL, "created_at" bigint NOT NULL, "updated_at" bigint NOT NULL, "email" varchar(255) NULL,
"upload_data_transfer" integer NOT NULL, "download_data_transfer" integer NOT NULL, "total_data_transfer" integer NOT NULL,
"used_upload_data_transfer" integer NOT NULL, "used_download_data_transfer" integer NOT NULL, "deleted_at" bigint NOT NULL,
"used_upload_data_transfer" bigint NOT NULL, "used_download_data_transfer" bigint NOT NULL, "deleted_at" bigint NOT NULL,
"first_download" bigint NOT NULL, "first_upload" bigint NOT NULL, "last_password_change" bigint NOT NULL, "role_id" integer NULL);
CREATE TABLE "{{groups_folders_mapping}}" ("id" integer NOT NULL PRIMARY KEY GENERATED ALWAYS AS IDENTITY, "group_id" integer NOT NULL,
"folder_id" integer NOT NULL, "virtual_path" text NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL);
@ -205,16 +205,10 @@ CREATE INDEX "{{prefix}}ip_lists_ipornet_idx" ON "{{ip_lists}}" ("ipornet");
CREATE INDEX "{{prefix}}ip_lists_updated_at_idx" ON "{{ip_lists}}" ("updated_at");
CREATE INDEX "{{prefix}}ip_lists_deleted_at_idx" ON "{{ip_lists}}" ("deleted_at");
CREATE INDEX "{{prefix}}ip_lists_first_last_idx" ON "{{ip_lists}}" ("first", "last");
INSERT INTO {{schema_version}} (version) VALUES (28);
INSERT INTO {{schema_version}} (version) VALUES (29);
`
// not supported in CockroachDB
ipListsLikeIndex = `CREATE INDEX "{{prefix}}ip_lists_ipornet_like_idx" ON "{{ip_lists}}" ("ipornet" varchar_pattern_ops);`
pgsqlV29SQL = `ALTER TABLE "{{users}}" ALTER COLUMN "used_download_data_transfer" TYPE bigint;
ALTER TABLE "{{users}}" ALTER COLUMN "used_upload_data_transfer" TYPE bigint;
`
pgsqlV29DownSQL = `ALTER TABLE "{{users}}" ALTER COLUMN "used_upload_data_transfer" TYPE integer;
ALTER TABLE "{{users}}" ALTER COLUMN "used_download_data_transfer" TYPE integer;
`
)
var (
@ -795,8 +789,8 @@ func (p *PGSQLProvider) initializeDatabase() error {
if errors.Is(err, sql.ErrNoRows) {
return errSchemaVersionEmpty
}
logger.InfoToConsole("creating initial database schema, version 28")
providerLog(logger.LevelInfo, "creating initial database schema, version 28")
logger.InfoToConsole("creating initial database schema, version 29")
providerLog(logger.LevelInfo, "creating initial database schema, version 29")
var initialSQL string
if config.Driver == CockroachDataProviderName {
initialSQL = sqlReplaceAll(pgsqlInitial)
@ -805,7 +799,7 @@ func (p *PGSQLProvider) initializeDatabase() error {
initialSQL = sqlReplaceAll(pgsqlInitial + ipListsLikeIndex)
}
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 28, true)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 29, true)
}
func (p *PGSQLProvider) migrateDatabase() error { //nolint:dupl
@ -818,13 +812,11 @@ func (p *PGSQLProvider) migrateDatabase() error { //nolint:dupl
case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version)
return ErrNoInitRequired
case version < 28:
case version < 29:
err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err)
return err
case version == 28:
return updatePGSQLDatabaseFrom28To29(p.dbHandle)
default:
if version > sqlDatabaseVersion {
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
@ -847,8 +839,6 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error {
}
switch dbVersion.Version {
case 29:
return downgradePGSQLDatabaseFrom29To28(p.dbHandle)
default:
return fmt.Errorf("database schema version not handled: %d", dbVersion.Version)
}
@ -886,19 +876,3 @@ func (p *PGSQLProvider) normalizeError(err error, fieldType int) error {
}
return err
}
func updatePGSQLDatabaseFrom28To29(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 28 -> 29")
providerLog(logger.LevelInfo, "updating database schema version: 28 -> 29")
sql := strings.ReplaceAll(pgsqlV29SQL, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 29, true)
}
func downgradePGSQLDatabaseFrom29To28(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 29 -> 28")
providerLog(logger.LevelInfo, "downgrading database schema version: 29 -> 28")
sql := strings.ReplaceAll(pgsqlV29DownSQL, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 28, false)
}

34
internal/dataprovider/sqlite.go
View file

@ -178,7 +178,7 @@ CREATE INDEX "{{prefix}}ip_lists_ip_type_idx" ON "{{ip_lists}}" ("ip_type");
CREATE INDEX "{{prefix}}ip_lists_ip_updated_at_idx" ON "{{ip_lists}}" ("updated_at");
CREATE INDEX "{{prefix}}ip_lists_ip_deleted_at_idx" ON "{{ip_lists}}" ("deleted_at");
CREATE INDEX "{{prefix}}ip_lists_first_last_idx" ON "{{ip_lists}}" ("first", "last");
INSERT INTO {{schema_version}} (version) VALUES (28);
INSERT INTO {{schema_version}} (version) VALUES (29);
`
)
@ -693,10 +693,10 @@ func (p *SQLiteProvider) initializeDatabase() error {
if errors.Is(err, sql.ErrNoRows) {
return errSchemaVersionEmpty
}
logger.InfoToConsole("creating initial database schema, version 28")
providerLog(logger.LevelInfo, "creating initial database schema, version 28")
logger.InfoToConsole("creating initial database schema, version 29")
providerLog(logger.LevelInfo, "creating initial database schema, version 29")
sql := sqlReplaceAll(sqliteInitialSQL)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 28, true)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 29, true)
}
func (p *SQLiteProvider) migrateDatabase() error { //nolint:dupl
@ -709,13 +709,11 @@ func (p *SQLiteProvider) migrateDatabase() error { //nolint:dupl
case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version)
return ErrNoInitRequired
case version < 28:
case version < 29:
err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err)
return err
case version == 28:
return updateSQLiteDatabaseFrom28To29(p.dbHandle)
default:
if version > sqlDatabaseVersion {
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
@ -738,8 +736,6 @@ func (p *SQLiteProvider) revertDatabase(targetVersion int) error {
}
switch dbVersion.Version {
case 29:
return downgradeSQLiteDatabaseFrom29To28(p.dbHandle)
default:
return fmt.Errorf("database schema version not handled: %d", dbVersion.Version)
}
@ -777,26 +773,6 @@ func (p *SQLiteProvider) normalizeError(err error, fieldType int) error {
return err
}
func updateSQLiteDatabaseFrom28To29(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 28 -> 29")
providerLog(logger.LevelInfo, "updating database schema version: 28 -> 29")
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 29)
}
func downgradeSQLiteDatabaseFrom29To28(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 29 -> 28")
providerLog(logger.LevelInfo, "downgrading database schema version: 29 -> 28")
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 28)
}
/*func setPragmaFK(dbHandle *sql.DB, value string) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel()