MySQL 8: Improve migrate command, ignore errors when dropping indexes
This commit is contained in:
parent
86c43159eb
commit
7e8974fd20
19 changed files with 369 additions and 145 deletions
|
@ -38,6 +38,7 @@ services:
|
|||
PHOTOPRISM_DATABASE_PASSWORD: "photoprism"
|
||||
PHOTOPRISM_TEST_DRIVER: "sqlite"
|
||||
PHOTOPRISM_TEST_DSN: ".test.db"
|
||||
PHOTOPRISM_TEST_DSN_MYSQL8: "root:photoprism@tcp(mysql:4001)/photoprism?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true"
|
||||
PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters)
|
||||
PHOTOPRISM_ASSETS_PATH: "/go/src/github.com/photoprism/photoprism/assets"
|
||||
PHOTOPRISM_STORAGE_PATH: "/go/src/github.com/photoprism/photoprism/storage"
|
||||
|
@ -157,6 +158,21 @@ services:
|
|||
MYSQL_PASSWORD: photoprism
|
||||
MYSQL_DATABASE: photoprism
|
||||
|
||||
## MySQL Database Server
|
||||
## Docs: https://dev.mysql.com/doc/refman/8.0/en/
|
||||
mysql:
|
||||
image: mysql:8
|
||||
command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120
|
||||
expose:
|
||||
- "4001" # Database port (internal)
|
||||
volumes:
|
||||
- "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql"
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: photoprism
|
||||
MYSQL_USER: photoprism
|
||||
MYSQL_PASSWORD: photoprism
|
||||
MYSQL_DATABASE: photoprism
|
||||
|
||||
## Dummy WebDAV Server
|
||||
dummy-webdav:
|
||||
image: photoprism/dummy-webdav:20211109
|
||||
|
|
|
@ -57,19 +57,6 @@ services:
|
|||
MYSQL_PASSWORD: photoprism
|
||||
MYSQL_DATABASE: photoprism
|
||||
|
||||
mysql-8:
|
||||
image: mysql:8
|
||||
command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120
|
||||
expose:
|
||||
- "4001" # Database port (internal)
|
||||
volumes:
|
||||
- "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql"
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: photoprism
|
||||
MYSQL_USER: photoprism
|
||||
MYSQL_PASSWORD: photoprism
|
||||
MYSQL_DATABASE: photoprism
|
||||
|
||||
networks:
|
||||
default:
|
||||
external:
|
|
@ -22,6 +22,7 @@ services:
|
|||
- "2343:2343" # Acceptance Test HTTP port (host:container)
|
||||
shm_size: "2gb"
|
||||
environment:
|
||||
PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters)
|
||||
PHOTOPRISM_UID: ${UID:-1000}
|
||||
PHOTOPRISM_GID: ${GID:-1000}
|
||||
PHOTOPRISM_SITE_URL: "http://localhost:2342/"
|
||||
|
@ -44,7 +45,7 @@ services:
|
|||
PHOTOPRISM_DATABASE_PASSWORD: "photoprism"
|
||||
PHOTOPRISM_TEST_DRIVER: "sqlite"
|
||||
PHOTOPRISM_TEST_DSN: ".test.db"
|
||||
PHOTOPRISM_ADMIN_PASSWORD: "photoprism" # The initial admin password (min 4 characters)
|
||||
PHOTOPRISM_TEST_DSN_MYSQL8: "root:photoprism@tcp(mysql:4001)/photoprism?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true"
|
||||
PHOTOPRISM_ASSETS_PATH: "/go/src/github.com/photoprism/photoprism/assets"
|
||||
PHOTOPRISM_STORAGE_PATH: "/go/src/github.com/photoprism/photoprism/storage"
|
||||
PHOTOPRISM_ORIGINALS_PATH: "/go/src/github.com/photoprism/photoprism/storage/originals"
|
||||
|
@ -103,6 +104,21 @@ services:
|
|||
MYSQL_PASSWORD: photoprism
|
||||
MYSQL_DATABASE: photoprism
|
||||
|
||||
## MySQL Database Server
|
||||
## Docs: https://dev.mysql.com/doc/refman/8.0/en/
|
||||
mysql:
|
||||
image: mysql:8
|
||||
command: mysqld --port=4001 --transaction-isolation=READ-COMMITTED --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --max-connections=512 --innodb-rollback-on-timeout=OFF --innodb-lock-wait-timeout=120
|
||||
expose:
|
||||
- "4001" # Database port (internal)
|
||||
volumes:
|
||||
- "./scripts/sql/init-test-databases.sql:/docker-entrypoint-initdb.d/init-test-databases.sql"
|
||||
environment:
|
||||
MYSQL_ROOT_PASSWORD: photoprism
|
||||
MYSQL_USER: photoprism
|
||||
MYSQL_PASSWORD: photoprism
|
||||
MYSQL_DATABASE: photoprism
|
||||
|
||||
## Dummy WebDAV Server
|
||||
dummy-webdav:
|
||||
image: photoprism/dummy-webdav:20211109
|
||||
|
|
|
@ -4,15 +4,26 @@ import (
|
|||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/photoprism/photoprism/internal/config"
|
||||
)
|
||||
|
||||
// MigrateCommand registers the migrate cli command.
|
||||
// MigrateCommand registers the "migrate" CLI command.
|
||||
var MigrateCommand = cli.Command{
|
||||
Name: "migrate",
|
||||
Usage: "Updates the index database schema",
|
||||
Name: "migrate",
|
||||
Usage: "Updates the index database schema",
|
||||
Flags: []cli.Flag{
|
||||
cli.BoolFlag{
|
||||
Name: "failed, f",
|
||||
Usage: "run previously failed migrations",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "trace, t",
|
||||
Usage: "show trace logs for debugging",
|
||||
},
|
||||
},
|
||||
Action: migrateAction,
|
||||
}
|
||||
|
||||
|
@ -29,15 +40,26 @@ func migrateAction(ctx *cli.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
defer conf.Shutdown()
|
||||
|
||||
if ctx.Bool("trace") {
|
||||
log.SetLevel(logrus.TraceLevel)
|
||||
log.Infoln("migrate: enabled trace mode")
|
||||
}
|
||||
|
||||
runFailed := ctx.Bool("failed")
|
||||
|
||||
if runFailed {
|
||||
log.Infoln("migrate: running previously failed migrations")
|
||||
}
|
||||
|
||||
log.Infoln("migrating database schema...")
|
||||
|
||||
conf.InitDb()
|
||||
conf.MigrateDb(runFailed)
|
||||
|
||||
elapsed := time.Since(start)
|
||||
|
||||
log.Infof("migration completed in %s", elapsed)
|
||||
|
||||
conf.Shutdown()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -16,11 +16,18 @@ import (
|
|||
// PlacesCommand registers the places subcommands.
|
||||
var PlacesCommand = cli.Command{
|
||||
Name: "places",
|
||||
Usage: "Geographic data subcommands",
|
||||
Usage: "Maps and location details subcommands",
|
||||
Subcommands: []cli.Command{
|
||||
{
|
||||
Name: "update",
|
||||
Usage: "Retrieves updated location details",
|
||||
Name: "update",
|
||||
Usage: "Retrieves updated location details",
|
||||
Flags: []cli.Flag{
|
||||
cli.BoolFlag{
|
||||
Name: "yes, y",
|
||||
Hidden: true,
|
||||
Usage: "assume \"yes\" as answer to all prompts and run non-interactively",
|
||||
},
|
||||
},
|
||||
Action: placesUpdateAction,
|
||||
},
|
||||
},
|
||||
|
@ -46,14 +53,16 @@ func placesUpdateAction(ctx *cli.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
confirmPrompt := promptui.Prompt{
|
||||
Label: "Interrupting the update may result in inconsistent location details. Proceed?",
|
||||
IsConfirm: true,
|
||||
}
|
||||
if !ctx.Bool("yes") {
|
||||
confirmPrompt := promptui.Prompt{
|
||||
Label: "Interrupting the update may result in inconsistent location details. Proceed?",
|
||||
IsConfirm: true,
|
||||
}
|
||||
|
||||
// Abort?
|
||||
if _, err := confirmPrompt.Run(); err != nil {
|
||||
return nil
|
||||
// Abort?
|
||||
if _, err := confirmPrompt.Run(); err != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
|
|
|
@ -9,6 +9,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/manifoldco/promptui"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/photoprism/photoprism/internal/config"
|
||||
|
@ -17,20 +18,27 @@ import (
|
|||
|
||||
// ResetCommand resets the index and removes sidecar files after confirmation.
|
||||
var ResetCommand = cli.Command{
|
||||
Name: "reset",
|
||||
Usage: "Resets the index and removes generated sidecar files",
|
||||
Name: "reset",
|
||||
Usage: "Resets the index and removes generated sidecar files",
|
||||
Flags: []cli.Flag{
|
||||
cli.BoolFlag{
|
||||
Name: "index, i",
|
||||
Usage: "reset index database only",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "trace, t",
|
||||
Usage: "show trace logs for debugging",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "yes, y",
|
||||
Usage: "assume \"yes\" as answer to all prompts and run non-interactively",
|
||||
},
|
||||
},
|
||||
Action: resetAction,
|
||||
}
|
||||
|
||||
// resetAction resets the index and removes sidecar files after confirmation.
|
||||
func resetAction(ctx *cli.Context) error {
|
||||
log.Warnf("YOU ARE ABOUT TO RESET THE INDEX AND REMOVE ALL JSON / YAML SIDECAR FILES")
|
||||
|
||||
removeIndexPrompt := promptui.Prompt{
|
||||
Label: "Reset index database including albums and metadata?",
|
||||
IsConfirm: true,
|
||||
}
|
||||
|
||||
conf := config.NewConfig(ctx)
|
||||
_, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
@ -39,101 +47,73 @@ func resetAction(ctx *cli.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
defer conf.Shutdown()
|
||||
|
||||
entity.SetDbProvider(conf)
|
||||
|
||||
if _, err := removeIndexPrompt.Run(); err == nil {
|
||||
start := time.Now()
|
||||
if !ctx.Bool("yes") {
|
||||
log.Warnf("This will delete and recreate your index database after confirmation")
|
||||
|
||||
tables := entity.Entities
|
||||
if !ctx.Bool("index") {
|
||||
log.Warnf("You will be asked next if you also want to remove all JSON and YAML backup files")
|
||||
}
|
||||
}
|
||||
|
||||
log.Infoln("dropping existing tables")
|
||||
tables.Drop()
|
||||
if ctx.Bool("trace") {
|
||||
log.SetLevel(logrus.TraceLevel)
|
||||
log.Infoln("reset: enabled trace mode")
|
||||
}
|
||||
|
||||
log.Infoln("restoring default schema")
|
||||
entity.MigrateDb(true)
|
||||
resetIndex := ctx.Bool("yes")
|
||||
|
||||
if conf.AdminPassword() != "" {
|
||||
log.Infoln("restoring initial admin password")
|
||||
entity.Admin.InitPassword(conf.AdminPassword())
|
||||
// Show prompt?
|
||||
if !resetIndex {
|
||||
removeIndexPrompt := promptui.Prompt{
|
||||
Label: "Delete and recreate index database?",
|
||||
IsConfirm: true,
|
||||
}
|
||||
|
||||
log.Infof("database reset completed in %s", time.Since(start))
|
||||
} else {
|
||||
log.Infof("keeping index database")
|
||||
if _, err := removeIndexPrompt.Run(); err == nil {
|
||||
resetIndex = true
|
||||
} else {
|
||||
log.Infof("keeping index database")
|
||||
}
|
||||
}
|
||||
|
||||
// Reset index?
|
||||
if resetIndex {
|
||||
resetIndexDb(conf)
|
||||
}
|
||||
|
||||
// Reset index only?
|
||||
if ctx.Bool("index") {
|
||||
return nil
|
||||
}
|
||||
|
||||
removeSidecarJsonPrompt := promptui.Prompt{
|
||||
Label: "Permanently delete existing JSON metadata sidecar files?",
|
||||
Label: "Delete all JSON metadata sidecar files?",
|
||||
IsConfirm: true,
|
||||
}
|
||||
|
||||
if _, err := removeSidecarJsonPrompt.Run(); err == nil {
|
||||
start := time.Now()
|
||||
|
||||
matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.json")
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(matches) > 0 {
|
||||
log.Infof("removing %d JSON metadata sidecar files", len(matches))
|
||||
|
||||
for _, name := range matches {
|
||||
if err := os.Remove(name); err != nil {
|
||||
fmt.Print("E")
|
||||
} else {
|
||||
fmt.Print(".")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("")
|
||||
|
||||
log.Infof("removed JSON metadata sidecar files [%s]", time.Since(start))
|
||||
} else {
|
||||
log.Infof("found no JSON metadata sidecar files")
|
||||
}
|
||||
resetSidecarJson(conf)
|
||||
} else {
|
||||
log.Infof("keeping JSON metadata sidecar files")
|
||||
}
|
||||
|
||||
removeSidecarYamlPrompt := promptui.Prompt{
|
||||
Label: "Permanently delete existing YAML metadata backups?",
|
||||
Label: "Delete all YAML metadata backup files?",
|
||||
IsConfirm: true,
|
||||
}
|
||||
|
||||
if _, err := removeSidecarYamlPrompt.Run(); err == nil {
|
||||
start := time.Now()
|
||||
|
||||
matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.yml")
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(matches) > 0 {
|
||||
log.Infof("%d YAML metadata backups will be removed", len(matches))
|
||||
|
||||
for _, name := range matches {
|
||||
if err := os.Remove(name); err != nil {
|
||||
fmt.Print("E")
|
||||
} else {
|
||||
fmt.Print(".")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("")
|
||||
|
||||
log.Infof("removed all YAML metadata backups [%s]", time.Since(start))
|
||||
} else {
|
||||
log.Infof("found no YAML metadata backups")
|
||||
}
|
||||
resetSidecarYaml(conf)
|
||||
} else {
|
||||
log.Infof("keeping YAML metadata backups")
|
||||
}
|
||||
|
||||
removeAlbumYamlPrompt := promptui.Prompt{
|
||||
Label: "Permanently delete existing YAML album backups?",
|
||||
Label: "Delete all YAML album backup files?",
|
||||
IsConfirm: true,
|
||||
}
|
||||
|
||||
|
@ -167,7 +147,85 @@ func resetAction(ctx *cli.Context) error {
|
|||
log.Infof("keeping YAML album backup files")
|
||||
}
|
||||
|
||||
conf.Shutdown()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// resetIndexDb resets the index database schema.
|
||||
func resetIndexDb(conf *config.Config) {
|
||||
start := time.Now()
|
||||
|
||||
tables := entity.Entities
|
||||
|
||||
log.Infoln("dropping existing tables")
|
||||
tables.Drop(conf.Db())
|
||||
|
||||
log.Infoln("restoring default schema")
|
||||
entity.MigrateDb(true, false)
|
||||
|
||||
if conf.AdminPassword() != "" {
|
||||
log.Infoln("restoring initial admin password")
|
||||
entity.Admin.InitPassword(conf.AdminPassword())
|
||||
}
|
||||
|
||||
log.Infof("database reset completed in %s", time.Since(start))
|
||||
}
|
||||
|
||||
// resetSidecarJson removes generated JSON sidecar files.
|
||||
func resetSidecarJson(conf *config.Config) {
|
||||
start := time.Now()
|
||||
|
||||
matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.json")
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("reset: %s (find json sidecar files)", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(matches) > 0 {
|
||||
log.Infof("removing %d JSON metadata sidecar files", len(matches))
|
||||
|
||||
for _, name := range matches {
|
||||
if err := os.Remove(name); err != nil {
|
||||
fmt.Print("E")
|
||||
} else {
|
||||
fmt.Print(".")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("")
|
||||
|
||||
log.Infof("removed JSON metadata sidecar files [%s]", time.Since(start))
|
||||
} else {
|
||||
log.Infof("found no JSON metadata sidecar files")
|
||||
}
|
||||
}
|
||||
|
||||
// resetSidecarYaml removes generated YAML sidecar files.
|
||||
func resetSidecarYaml(conf *config.Config) {
|
||||
start := time.Now()
|
||||
|
||||
matches, err := filepath.Glob(regexp.QuoteMeta(conf.SidecarPath()) + "/**/*.yml")
|
||||
|
||||
if err != nil {
|
||||
log.Errorf("reset: %s (find yaml sidecar files)", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(matches) > 0 {
|
||||
log.Infof("%d YAML metadata backups will be removed", len(matches))
|
||||
|
||||
for _, name := range matches {
|
||||
if err := os.Remove(name); err != nil {
|
||||
fmt.Print("E")
|
||||
} else {
|
||||
fmt.Print(".")
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("")
|
||||
|
||||
log.Infof("removed all YAML metadata backups [%s]", time.Since(start))
|
||||
} else {
|
||||
log.Infof("found no YAML metadata backups")
|
||||
}
|
||||
}
|
||||
|
|
|
@ -12,7 +12,6 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize/english"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
|
||||
"github.com/photoprism/photoprism/internal/config"
|
||||
|
@ -151,7 +150,7 @@ func restoreAction(ctx *cli.Context) error {
|
|||
)
|
||||
case config.SQLite:
|
||||
log.Infoln("dropping existing tables")
|
||||
tables.Drop()
|
||||
tables.Drop(conf.Db())
|
||||
cmd = exec.Command(
|
||||
conf.SqliteBin(),
|
||||
conf.DatabaseDsn(),
|
||||
|
|
|
@ -232,11 +232,16 @@ func (c *Config) SetDbOptions() {
|
|||
}
|
||||
}
|
||||
|
||||
// InitDb will initialize the database connection and schema.
|
||||
// InitDb initializes the database without running previously failed migrations.
|
||||
func (c *Config) InitDb() {
|
||||
c.MigrateDb(false)
|
||||
}
|
||||
|
||||
// MigrateDb initializes the database and migrates the schema if needed.
|
||||
func (c *Config) MigrateDb(runFailed bool) {
|
||||
c.SetDbOptions()
|
||||
entity.SetDbProvider(c)
|
||||
entity.MigrateDb(false)
|
||||
entity.MigrateDb(true, runFailed)
|
||||
|
||||
entity.Admin.InitPassword(c.AdminPassword())
|
||||
|
||||
|
|
|
@ -13,9 +13,9 @@ func CreateDefaultFixtures() {
|
|||
|
||||
// ResetTestFixtures re-creates registered database tables and inserts test fixtures.
|
||||
func ResetTestFixtures() {
|
||||
Entities.Migrate()
|
||||
Entities.WaitForMigration()
|
||||
Entities.Truncate()
|
||||
Entities.Migrate(Db(), false)
|
||||
Entities.WaitForMigration(Db())
|
||||
Entities.Truncate(Db())
|
||||
|
||||
CreateDefaultFixtures()
|
||||
|
||||
|
|
|
@ -1,13 +1,13 @@
|
|||
package entity
|
||||
|
||||
// MigrateDb creates database tables and inserts default fixtures as needed.
|
||||
func MigrateDb(dropDeprecated bool) {
|
||||
func MigrateDb(dropDeprecated, runFailed bool) {
|
||||
if dropDeprecated {
|
||||
DeprecatedTables.Drop()
|
||||
DeprecatedTables.Drop(Db())
|
||||
}
|
||||
|
||||
Entities.Migrate()
|
||||
Entities.WaitForMigration()
|
||||
Entities.Migrate(Db(), runFailed)
|
||||
Entities.WaitForMigration(Db())
|
||||
|
||||
CreateDefaultFixtures()
|
||||
}
|
||||
|
|
51
internal/entity/db_mysql8_test.go
Normal file
51
internal/entity/db_mysql8_test.go
Normal file
|
@ -0,0 +1,51 @@
|
|||
package entity
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
)
|
||||
|
||||
func TestMySQL8(t *testing.T) {
|
||||
dbDriver := MySQL
|
||||
dbDsn := os.Getenv("PHOTOPRISM_TEST_DSN_MYSQL8")
|
||||
|
||||
db, err := gorm.Open(dbDriver, dbDsn)
|
||||
|
||||
if err != nil || db == nil {
|
||||
for i := 1; i <= 5; i++ {
|
||||
db, err = gorm.Open(dbDriver, dbDsn)
|
||||
|
||||
if db != nil && err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
if err != nil || db == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
defer db.Close()
|
||||
|
||||
db.LogMode(false)
|
||||
|
||||
DeprecatedTables.Drop(db)
|
||||
Entities.Drop(db)
|
||||
|
||||
// First migration.
|
||||
Entities.Migrate(db, false)
|
||||
Entities.WaitForMigration(db)
|
||||
|
||||
// Second migration.
|
||||
Entities.Migrate(db, false)
|
||||
Entities.WaitForMigration(db)
|
||||
|
||||
// Third migration with force flag.
|
||||
Entities.Migrate(db, true)
|
||||
Entities.WaitForMigration(db)
|
||||
}
|
|
@ -4,6 +4,8 @@ import (
|
|||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
|
||||
"github.com/photoprism/photoprism/internal/migrate"
|
||||
"github.com/photoprism/photoprism/pkg/txt"
|
||||
)
|
||||
|
@ -44,7 +46,7 @@ var Entities = Tables{
|
|||
}
|
||||
|
||||
// WaitForMigration waits for the database migration to be successful.
|
||||
func (list Tables) WaitForMigration() {
|
||||
func (list Tables) WaitForMigration(db *gorm.DB) {
|
||||
type RowCount struct {
|
||||
Count int
|
||||
}
|
||||
|
@ -53,7 +55,7 @@ func (list Tables) WaitForMigration() {
|
|||
for name := range list {
|
||||
for i := 0; i <= attempts; i++ {
|
||||
count := RowCount{}
|
||||
if err := Db().Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil {
|
||||
if err := db.Raw(fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", name)).Scan(&count).Error; err == nil {
|
||||
log.Tracef("entity: %s migrated", txt.Quote(name))
|
||||
break
|
||||
} else {
|
||||
|
@ -70,9 +72,9 @@ func (list Tables) WaitForMigration() {
|
|||
}
|
||||
|
||||
// Truncate removes all data from tables without dropping them.
|
||||
func (list Tables) Truncate() {
|
||||
func (list Tables) Truncate(db *gorm.DB) {
|
||||
for name := range list {
|
||||
if err := Db().Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil {
|
||||
if err := db.Exec(fmt.Sprintf("DELETE FROM %s WHERE 1", name)).Error; err == nil {
|
||||
// log.Debugf("entity: removed all data from %s", name)
|
||||
break
|
||||
} else if err.Error() != "record not found" {
|
||||
|
@ -82,29 +84,29 @@ func (list Tables) Truncate() {
|
|||
}
|
||||
|
||||
// Migrate migrates all database tables of registered entities.
|
||||
func (list Tables) Migrate() {
|
||||
func (list Tables) Migrate(db *gorm.DB, runFailed bool) {
|
||||
for name, entity := range list {
|
||||
if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
|
||||
if err := db.AutoMigrate(entity).Error; err != nil {
|
||||
log.Debugf("entity: %s (waiting 1s)", err.Error())
|
||||
|
||||
time.Sleep(time.Second)
|
||||
|
||||
if err := UnscopedDb().AutoMigrate(entity).Error; err != nil {
|
||||
if err := db.AutoMigrate(entity).Error; err != nil {
|
||||
log.Errorf("entity: failed migrating %s", txt.Quote(name))
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := migrate.Auto(Db()); err != nil {
|
||||
if err := migrate.Auto(db, runFailed); err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Drop drops all database tables of registered entities.
|
||||
func (list Tables) Drop() {
|
||||
func (list Tables) Drop(db *gorm.DB) {
|
||||
for _, entity := range list {
|
||||
if err := UnscopedDb().DropTableIfExists(entity).Error; err != nil {
|
||||
if err := db.DropTableIfExists(entity).Error; err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,12 +1,14 @@
|
|||
package entity
|
||||
|
||||
import "github.com/jinzhu/gorm"
|
||||
|
||||
// Deprecated represents a list of deprecated database tables.
|
||||
type Deprecated []string
|
||||
|
||||
// Drop drops all deprecated tables.
|
||||
func (list Deprecated) Drop() {
|
||||
func (list Deprecated) Drop(db *gorm.DB) {
|
||||
for _, tableName := range list {
|
||||
if err := UnscopedDb().DropTableIfExists(tableName).Error; err != nil {
|
||||
if err := db.DropTableIfExists(tableName).Error; err != nil {
|
||||
log.Debugf("drop %s: %s", tableName, err)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -7,7 +7,7 @@ import (
|
|||
)
|
||||
|
||||
// Auto automatically migrates the database provided.
|
||||
func Auto(db *gorm.DB) error {
|
||||
func Auto(db *gorm.DB, runFailed bool) error {
|
||||
if db == nil {
|
||||
return fmt.Errorf("migrate: database connection required")
|
||||
}
|
||||
|
@ -23,7 +23,7 @@ func Auto(db *gorm.DB) error {
|
|||
}
|
||||
|
||||
if migrations, ok := Dialects[name]; ok && len(migrations) > 0 {
|
||||
migrations.Start(db)
|
||||
migrations.Start(db, runFailed)
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("migrate: no migrations found for %s", name)
|
||||
|
|
|
@ -5,11 +5,11 @@ var DialectMySQL = Migrations{
|
|||
{
|
||||
ID: "20211121-094727",
|
||||
Dialect: "mysql",
|
||||
Statements: []string{"DROP INDEX IF EXISTS uix_places_place_label ON `places`;"},
|
||||
Statements: []string{"DROP INDEX uix_places_place_label ON `places`;"},
|
||||
},
|
||||
{
|
||||
ID: "20211124-120008",
|
||||
Dialect: "mysql",
|
||||
Statements: []string{"DROP INDEX IF EXISTS idx_places_place_label ON `places`;", "DROP INDEX IF EXISTS uix_places_label ON `places`;"},
|
||||
Statements: []string{"DROP INDEX idx_places_place_label ON `places`;", "DROP INDEX uix_places_label ON `places`;"},
|
||||
},
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package migrate
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jinzhu/gorm"
|
||||
|
@ -29,6 +30,7 @@ func (m *Migration) Fail(err error, db *gorm.DB) {
|
|||
}
|
||||
|
||||
m.Error = err.Error()
|
||||
|
||||
db.Model(m).Updates(Values{"Error": m.Error})
|
||||
}
|
||||
|
||||
|
@ -41,7 +43,11 @@ func (m *Migration) Finish(db *gorm.DB) error {
|
|||
func (m *Migration) Execute(db *gorm.DB) error {
|
||||
for _, s := range m.Statements {
|
||||
if err := db.Exec(s).Error; err != nil {
|
||||
return err
|
||||
if strings.HasPrefix(s, "DROP ") && strings.Contains(err.Error(), "DROP") {
|
||||
log.Tracef("migrate: %s (drop statement)", err)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1,33 +1,84 @@
|
|||
package migrate
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize/english"
|
||||
"github.com/jinzhu/gorm"
|
||||
)
|
||||
|
||||
// Migrations represents a sorted list of migrations.
|
||||
type Migrations []Migration
|
||||
|
||||
// MigrationMap represents a map of migrations.
|
||||
type MigrationMap map[string]Migration
|
||||
|
||||
// Existing finds and returns previously executed database schema migrations.
|
||||
func Existing(db *gorm.DB) MigrationMap {
|
||||
result := make(MigrationMap)
|
||||
dialect := db.Dialect().GetName()
|
||||
|
||||
stmt := db.Model(Migration{}).Where("dialect = ?", dialect)
|
||||
stmt = stmt.Select("id, dialect, error, source, started_at, finished_at")
|
||||
|
||||
rows, err := stmt.Rows()
|
||||
|
||||
if err != nil {
|
||||
log.Warnf("migrate: %s (find existing)", err)
|
||||
return result
|
||||
}
|
||||
|
||||
defer func(rows *sql.Rows) {
|
||||
err = rows.Close()
|
||||
}(rows)
|
||||
|
||||
for rows.Next() {
|
||||
m := Migration{}
|
||||
|
||||
if err = rows.Scan(&m.ID, &m.Dialect, &m.Error, &m.Source, &m.StartedAt, &m.FinishedAt); err != nil {
|
||||
log.Warnf("migrate: %s (scan existing)", err)
|
||||
return result
|
||||
}
|
||||
|
||||
result[m.ID] = m
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// Start runs all migrations that haven't been executed yet.
|
||||
func (m *Migrations) Start(db *gorm.DB) {
|
||||
func (m *Migrations) Start(db *gorm.DB, runFailed bool) {
|
||||
// Find previously executed migrations.
|
||||
executed := Existing(db)
|
||||
|
||||
log.Debugf("migrate: found %s", english.Plural(len(executed), "previous migration", "previous migrations"))
|
||||
|
||||
for _, migration := range *m {
|
||||
start := time.Now()
|
||||
|
||||
migration.StartedAt = start.UTC().Round(time.Second)
|
||||
|
||||
// Continue if already executed.
|
||||
if err := db.Create(migration).Error; err != nil {
|
||||
// Already executed?
|
||||
if done, ok := executed[migration.ID]; ok {
|
||||
// Try to run failed migrations again?
|
||||
if !runFailed || done.Error == "" {
|
||||
log.Debugf("migrate: %s skipped", migration.ID)
|
||||
continue
|
||||
}
|
||||
} else if err := db.Create(migration).Error; err != nil {
|
||||
// Should not happen.
|
||||
log.Warnf("migrate: creating %s failed with %s [%s]", migration.ID, err, time.Since(start))
|
||||
continue
|
||||
}
|
||||
|
||||
// Run migration.
|
||||
if err := migration.Execute(db); err != nil {
|
||||
migration.Fail(err, db)
|
||||
log.Errorf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start))
|
||||
log.Errorf("migrate: executing %s failed with %s [%s]", migration.ID, err, time.Since(start))
|
||||
} else if err = migration.Finish(db); err != nil {
|
||||
log.Warnf("migration %s failed: %s [%s]", migration.ID, err, time.Since(start))
|
||||
log.Warnf("migrate: updating %s failed with %s [%s]", migration.ID, err, time.Since(start))
|
||||
} else {
|
||||
log.Infof("migration %s successful [%s]", migration.ID, time.Since(start))
|
||||
log.Infof("migrate: %s successful [%s]", migration.ID, time.Since(start))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1 +1 @@
|
|||
DROP INDEX IF EXISTS uix_places_place_label ON `places`;
|
||||
DROP INDEX uix_places_place_label ON `places`;
|
|
@ -1,2 +1,2 @@
|
|||
DROP INDEX IF EXISTS idx_places_place_label ON `places`;
|
||||
DROP INDEX IF EXISTS uix_places_label ON `places`;
|
||||
DROP INDEX idx_places_place_label ON `places`;
|
||||
DROP INDEX uix_places_label ON `places`;
|
Loading…
Add table
Reference in a new issue