From c5c42f072b155922b4c502755f849b22d0c13561 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 15 Jun 2024 16:02:09 +0200 Subject: [PATCH] squash database migrations Signed-off-by: Nicola Murino --- internal/cmd/revertprovider.go | 6 ++--- internal/dataprovider/bolt.go | 16 +++-------- internal/dataprovider/dataprovider.go | 7 ++--- internal/dataprovider/mysql.go | 38 +++++---------------------- internal/dataprovider/pgsql.go | 38 +++++---------------------- internal/dataprovider/sqlite.go | 34 ++++-------------------- 6 files changed, 29 insertions(+), 110 deletions(-) diff --git a/internal/cmd/revertprovider.go b/internal/cmd/revertprovider.go index 64de3529..f28fc8a3 100644 --- a/internal/cmd/revertprovider.go +++ b/internal/cmd/revertprovider.go @@ -40,8 +40,8 @@ Please take a look at the usage below to customize the options.`, Run: func(_ *cobra.Command, _ []string) { logger.DisableLogger() logger.EnableConsoleLogger(zerolog.DebugLevel) - if revertProviderTargetVersion != 28 { - logger.WarnToConsole("Unsupported target version, 28 is the only supported one") + if revertProviderTargetVersion != 29 { + logger.WarnToConsole("Unsupported target version, 29 is the only supported one") os.Exit(1) } configDir = util.CleanDirInput(configDir) @@ -71,7 +71,7 @@ Please take a look at the usage below to customize the options.`, func init() { addConfigFlags(revertProviderCmd) - revertProviderCmd.Flags().IntVar(&revertProviderTargetVersion, "to-version", 28, `28 means the version supported in v2.5.x`) + revertProviderCmd.Flags().IntVar(&revertProviderTargetVersion, "to-version", 29, `29 means the version supported in v2.6.x`) rootCmd.AddCommand(revertProviderCmd) } diff --git a/internal/dataprovider/bolt.go b/internal/dataprovider/bolt.go index 01ad0b3e..59b9d095 100644 --- a/internal/dataprovider/bolt.go +++ b/internal/dataprovider/bolt.go @@ -3134,15 +3134,11 @@ func (p *BoltProvider) migrateDatabase() error { case version == boltDatabaseVersion: providerLog(logger.LevelDebug, "bolt database is up to date, current version: %d", version) return ErrNoInitRequired - case version < 28: + case version < 29: err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version) providerLog(logger.LevelError, "%v", err) logger.ErrorToConsole("%v", err) return err - case version == 28: - logger.InfoToConsole("updating database schema version: %d -> 29", version) - providerLog(logger.LevelInfo, "updating database schema version: %d -> 29", version) - return updateBoltDatabaseVersion(p.dbHandle, 29) default: if version > boltDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -3164,10 +3160,6 @@ func (p *BoltProvider) revertDatabase(targetVersion int) error { //nolint:gocycl return errors.New("current version match target version, nothing to do") } switch dbVersion.Version { - case 29: - logger.InfoToConsole("downgrading database schema version: %d -> 28", dbVersion.Version) - providerLog(logger.LevelInfo, "downgrading database schema version: %d -> 28", dbVersion.Version) - return updateBoltDatabaseVersion(p.dbHandle, 28) default: return fmt.Errorf("database schema version not handled: %v", dbVersion.Version) } @@ -3899,7 +3891,7 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) { v := bucket.Get(dbVersionKey) if v == nil { dbVersion = schemaVersion{ - Version: 28, + Version: 29, } return nil } @@ -3908,7 +3900,7 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) { return dbVersion, err } -func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error { +/*func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error { err := dbHandle.Update(func(tx *bolt.Tx) error { bucket := tx.Bucket(dbVersionBucket) if bucket == nil { @@ -3924,4 +3916,4 @@ func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error { return bucket.Put(dbVersionKey, buf) }) return err -} +}*/ diff --git a/internal/dataprovider/dataprovider.go b/internal/dataprovider/dataprovider.go index 583e8cc6..6b420c82 100644 --- a/internal/dataprovider/dataprovider.go +++ b/internal/dataprovider/dataprovider.go @@ -929,12 +929,13 @@ func checkDatabase(checkAdmins bool) error { if config.UpdateMode == 0 { err := provider.initializeDatabase() if err != nil && err != ErrNoInitRequired { - logger.WarnToConsole("Unable to initialize data provider: %v", err) - providerLog(logger.LevelError, "Unable to initialize data provider: %v", err) + logger.WarnToConsole("unable to initialize data provider: %v", err) + providerLog(logger.LevelError, "unable to initialize data provider: %v", err) return err } if err == nil { - logger.DebugToConsole("Data provider successfully initialized") + logger.DebugToConsole("data provider successfully initialized") + providerLog(logger.LevelInfo, "data provider successfully initialized") } err = provider.migrateDatabase() if err != nil && err != ErrNoInitRequired { diff --git a/internal/dataprovider/mysql.go b/internal/dataprovider/mysql.go index 659637d9..74f7b39a 100644 --- a/internal/dataprovider/mysql.go +++ b/internal/dataprovider/mysql.go @@ -95,8 +95,8 @@ const ( "`last_login` bigint NOT NULL, `filters` longtext NULL, `filesystem` longtext NULL, `additional_info` longtext NULL, " + "`created_at` bigint NOT NULL, `updated_at` bigint NOT NULL, `email` varchar(255) NULL, " + "`upload_data_transfer` integer NOT NULL, `download_data_transfer` integer NOT NULL, " + - "`total_data_transfer` integer NOT NULL, `used_upload_data_transfer` integer NOT NULL, " + - "`used_download_data_transfer` integer NOT NULL, `deleted_at` bigint NOT NULL, `first_download` bigint NOT NULL, " + + "`total_data_transfer` integer NOT NULL, `used_upload_data_transfer` bigint NOT NULL, " + + "`used_download_data_transfer` bigint NOT NULL, `deleted_at` bigint NOT NULL, `first_download` bigint NOT NULL, " + "`first_upload` bigint NOT NULL, `last_password_change` bigint NOT NULL, `role_id` integer NULL);" + "CREATE TABLE `{{groups_folders_mapping}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, " + "`group_id` integer NOT NULL, `folder_id` integer NOT NULL, " + @@ -193,11 +193,7 @@ const ( "CREATE INDEX `{{prefix}}ip_lists_updated_at_idx` ON `{{ip_lists}}` (`updated_at`);" + "CREATE INDEX `{{prefix}}ip_lists_deleted_at_idx` ON `{{ip_lists}}` (`deleted_at`);" + "CREATE INDEX `{{prefix}}ip_lists_first_last_idx` ON `{{ip_lists}}` (`first`, `last`);" + - "INSERT INTO {{schema_version}} (version) VALUES (28);" - mysqlV29SQL = "ALTER TABLE `{{users}}` MODIFY `used_download_data_transfer` bigint NOT NULL;" + - "ALTER TABLE `{{users}}` MODIFY `used_upload_data_transfer` bigint NOT NULL;" - mysqlV29DownSQL = "ALTER TABLE `{{users}}` MODIFY `used_upload_data_transfer` integer NOT NULL;" + - "ALTER TABLE `{{users}}` MODIFY `used_download_data_transfer` integer NOT NULL;" + "INSERT INTO {{schema_version}} (version) VALUES (29);" ) // MySQLProvider defines the auth provider for MySQL/MariaDB database @@ -776,11 +772,11 @@ func (p *MySQLProvider) initializeDatabase() error { if errors.Is(err, sql.ErrNoRows) { return errSchemaVersionEmpty } - logger.InfoToConsole("creating initial database schema, version 28") - providerLog(logger.LevelInfo, "creating initial database schema, version 28") + logger.InfoToConsole("creating initial database schema, version 29") + providerLog(logger.LevelInfo, "creating initial database schema, version 29") initialSQL := sqlReplaceAll(mysqlInitialSQL) - return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, strings.Split(initialSQL, ";"), 28, true) + return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, strings.Split(initialSQL, ";"), 29, true) } func (p *MySQLProvider) migrateDatabase() error { @@ -793,13 +789,11 @@ func (p *MySQLProvider) migrateDatabase() error { case version == sqlDatabaseVersion: providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version) return ErrNoInitRequired - case version < 28: + case version < 29: err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version) providerLog(logger.LevelError, "%v", err) logger.ErrorToConsole("%v", err) return err - case version == 28: - return updateMySQLDatabaseFrom28To29(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -822,8 +816,6 @@ func (p *MySQLProvider) revertDatabase(targetVersion int) error { } switch dbVersion.Version { - case 29: - return downgradeMySQLDatabaseFrom29To28(p.dbHandle) default: return fmt.Errorf("database schema version not handled: %d", dbVersion.Version) } @@ -861,19 +853,3 @@ func (p *MySQLProvider) normalizeError(err error, fieldType int) error { } return err } - -func updateMySQLDatabaseFrom28To29(dbHandle *sql.DB) error { - logger.InfoToConsole("updating database schema version: 28 -> 29") - providerLog(logger.LevelInfo, "updating database schema version: 28 -> 29") - - sql := strings.ReplaceAll(mysqlV29SQL, "{{users}}", sqlTableUsers) - return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 29, true) -} - -func downgradeMySQLDatabaseFrom29To28(dbHandle *sql.DB) error { - logger.InfoToConsole("downgrading database schema version: 29 -> 28") - providerLog(logger.LevelInfo, "downgrading database schema version: 29 -> 28") - - sql := strings.ReplaceAll(mysqlV29DownSQL, "{{users}}", sqlTableUsers) - return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 28, false) -} diff --git a/internal/dataprovider/pgsql.go b/internal/dataprovider/pgsql.go index 3e7d69d2..75ddd11f 100644 --- a/internal/dataprovider/pgsql.go +++ b/internal/dataprovider/pgsql.go @@ -95,7 +95,7 @@ CREATE TABLE "{{users}}" ("id" integer NOT NULL PRIMARY KEY GENERATED ALWAYS AS "download_bandwidth" integer NOT NULL, "last_login" bigint NOT NULL, "filters" text NULL, "filesystem" text NULL, "additional_info" text NULL, "created_at" bigint NOT NULL, "updated_at" bigint NOT NULL, "email" varchar(255) NULL, "upload_data_transfer" integer NOT NULL, "download_data_transfer" integer NOT NULL, "total_data_transfer" integer NOT NULL, -"used_upload_data_transfer" integer NOT NULL, "used_download_data_transfer" integer NOT NULL, "deleted_at" bigint NOT NULL, +"used_upload_data_transfer" bigint NOT NULL, "used_download_data_transfer" bigint NOT NULL, "deleted_at" bigint NOT NULL, "first_download" bigint NOT NULL, "first_upload" bigint NOT NULL, "last_password_change" bigint NOT NULL, "role_id" integer NULL); CREATE TABLE "{{groups_folders_mapping}}" ("id" integer NOT NULL PRIMARY KEY GENERATED ALWAYS AS IDENTITY, "group_id" integer NOT NULL, "folder_id" integer NOT NULL, "virtual_path" text NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL); @@ -205,16 +205,10 @@ CREATE INDEX "{{prefix}}ip_lists_ipornet_idx" ON "{{ip_lists}}" ("ipornet"); CREATE INDEX "{{prefix}}ip_lists_updated_at_idx" ON "{{ip_lists}}" ("updated_at"); CREATE INDEX "{{prefix}}ip_lists_deleted_at_idx" ON "{{ip_lists}}" ("deleted_at"); CREATE INDEX "{{prefix}}ip_lists_first_last_idx" ON "{{ip_lists}}" ("first", "last"); -INSERT INTO {{schema_version}} (version) VALUES (28); +INSERT INTO {{schema_version}} (version) VALUES (29); ` // not supported in CockroachDB ipListsLikeIndex = `CREATE INDEX "{{prefix}}ip_lists_ipornet_like_idx" ON "{{ip_lists}}" ("ipornet" varchar_pattern_ops);` - pgsqlV29SQL = `ALTER TABLE "{{users}}" ALTER COLUMN "used_download_data_transfer" TYPE bigint; -ALTER TABLE "{{users}}" ALTER COLUMN "used_upload_data_transfer" TYPE bigint; -` - pgsqlV29DownSQL = `ALTER TABLE "{{users}}" ALTER COLUMN "used_upload_data_transfer" TYPE integer; -ALTER TABLE "{{users}}" ALTER COLUMN "used_download_data_transfer" TYPE integer; -` ) var ( @@ -795,8 +789,8 @@ func (p *PGSQLProvider) initializeDatabase() error { if errors.Is(err, sql.ErrNoRows) { return errSchemaVersionEmpty } - logger.InfoToConsole("creating initial database schema, version 28") - providerLog(logger.LevelInfo, "creating initial database schema, version 28") + logger.InfoToConsole("creating initial database schema, version 29") + providerLog(logger.LevelInfo, "creating initial database schema, version 29") var initialSQL string if config.Driver == CockroachDataProviderName { initialSQL = sqlReplaceAll(pgsqlInitial) @@ -805,7 +799,7 @@ func (p *PGSQLProvider) initializeDatabase() error { initialSQL = sqlReplaceAll(pgsqlInitial + ipListsLikeIndex) } - return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 28, true) + return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 29, true) } func (p *PGSQLProvider) migrateDatabase() error { //nolint:dupl @@ -818,13 +812,11 @@ func (p *PGSQLProvider) migrateDatabase() error { //nolint:dupl case version == sqlDatabaseVersion: providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version) return ErrNoInitRequired - case version < 28: + case version < 29: err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version) providerLog(logger.LevelError, "%v", err) logger.ErrorToConsole("%v", err) return err - case version == 28: - return updatePGSQLDatabaseFrom28To29(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -847,8 +839,6 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error { } switch dbVersion.Version { - case 29: - return downgradePGSQLDatabaseFrom29To28(p.dbHandle) default: return fmt.Errorf("database schema version not handled: %d", dbVersion.Version) } @@ -886,19 +876,3 @@ func (p *PGSQLProvider) normalizeError(err error, fieldType int) error { } return err } - -func updatePGSQLDatabaseFrom28To29(dbHandle *sql.DB) error { - logger.InfoToConsole("updating database schema version: 28 -> 29") - providerLog(logger.LevelInfo, "updating database schema version: 28 -> 29") - - sql := strings.ReplaceAll(pgsqlV29SQL, "{{users}}", sqlTableUsers) - return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 29, true) -} - -func downgradePGSQLDatabaseFrom29To28(dbHandle *sql.DB) error { - logger.InfoToConsole("downgrading database schema version: 29 -> 28") - providerLog(logger.LevelInfo, "downgrading database schema version: 29 -> 28") - - sql := strings.ReplaceAll(pgsqlV29DownSQL, "{{users}}", sqlTableUsers) - return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 28, false) -} diff --git a/internal/dataprovider/sqlite.go b/internal/dataprovider/sqlite.go index 4a21c21c..82f298e2 100644 --- a/internal/dataprovider/sqlite.go +++ b/internal/dataprovider/sqlite.go @@ -178,7 +178,7 @@ CREATE INDEX "{{prefix}}ip_lists_ip_type_idx" ON "{{ip_lists}}" ("ip_type"); CREATE INDEX "{{prefix}}ip_lists_ip_updated_at_idx" ON "{{ip_lists}}" ("updated_at"); CREATE INDEX "{{prefix}}ip_lists_ip_deleted_at_idx" ON "{{ip_lists}}" ("deleted_at"); CREATE INDEX "{{prefix}}ip_lists_first_last_idx" ON "{{ip_lists}}" ("first", "last"); -INSERT INTO {{schema_version}} (version) VALUES (28); +INSERT INTO {{schema_version}} (version) VALUES (29); ` ) @@ -693,10 +693,10 @@ func (p *SQLiteProvider) initializeDatabase() error { if errors.Is(err, sql.ErrNoRows) { return errSchemaVersionEmpty } - logger.InfoToConsole("creating initial database schema, version 28") - providerLog(logger.LevelInfo, "creating initial database schema, version 28") + logger.InfoToConsole("creating initial database schema, version 29") + providerLog(logger.LevelInfo, "creating initial database schema, version 29") sql := sqlReplaceAll(sqliteInitialSQL) - return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 28, true) + return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 29, true) } func (p *SQLiteProvider) migrateDatabase() error { //nolint:dupl @@ -709,13 +709,11 @@ func (p *SQLiteProvider) migrateDatabase() error { //nolint:dupl case version == sqlDatabaseVersion: providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version) return ErrNoInitRequired - case version < 28: + case version < 29: err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version) providerLog(logger.LevelError, "%v", err) logger.ErrorToConsole("%v", err) return err - case version == 28: - return updateSQLiteDatabaseFrom28To29(p.dbHandle) default: if version > sqlDatabaseVersion { providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version, @@ -738,8 +736,6 @@ func (p *SQLiteProvider) revertDatabase(targetVersion int) error { } switch dbVersion.Version { - case 29: - return downgradeSQLiteDatabaseFrom29To28(p.dbHandle) default: return fmt.Errorf("database schema version not handled: %d", dbVersion.Version) } @@ -777,26 +773,6 @@ func (p *SQLiteProvider) normalizeError(err error, fieldType int) error { return err } -func updateSQLiteDatabaseFrom28To29(dbHandle *sql.DB) error { - logger.InfoToConsole("updating database schema version: 28 -> 29") - providerLog(logger.LevelInfo, "updating database schema version: 28 -> 29") - - ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) - defer cancel() - - return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 29) -} - -func downgradeSQLiteDatabaseFrom29To28(dbHandle *sql.DB) error { - logger.InfoToConsole("downgrading database schema version: 29 -> 28") - providerLog(logger.LevelInfo, "downgrading database schema version: 29 -> 28") - - ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) - defer cancel() - - return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 28) -} - /*func setPragmaFK(dbHandle *sql.DB, value string) error { ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) defer cancel()