beenull/photoprism
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Index: Improve save / update functions, remove orphans

Browse source
This commit is contained in:
Michael Mayer 2021-02-06 16:30:30 +01:00
parent 31fb402a28
commit 1bfb2e1774
28 changed files with 456 additions and 282 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
internal/api/account.go
View file

@ -124,7 +124,7 @@ func GetAccountFolders(router *gin.RouterGroup) {
if cacheData, ok := cache.Get(cacheKey); ok {
cached := cacheData.(fs.FileInfos)
log.Debugf("cache hit for %s [%s]", cacheKey, time.Since(start))
log.Debugf("api: cache hit for %s [%s]", cacheKey, time.Since(start))
c.JSON(http.StatusOK, cached)
return

4
internal/api/covers.go
View file

@ -50,7 +50,7 @@ func AlbumCover(router *gin.RouterGroup) {
cacheKey := CacheKey(albumCover, uid, typeName)
if cacheData, ok := cache.Get(cacheKey); ok {
log.Debugf("cache hit for %s [%s]", cacheKey, time.Since(start))
log.Debugf("api: cache hit for %s [%s]", cacheKey, time.Since(start))
cached := cacheData.(ThumbCache)
@ -160,7 +160,7 @@ func LabelCover(router *gin.RouterGroup) {
cacheKey := CacheKey(labelCover, uid, typeName)
if cacheData, ok := cache.Get(cacheKey); ok {
log.Debugf("cache hit for %s [%s]", cacheKey, time.Since(start))
log.Debugf("api: cache hit for %s [%s]", cacheKey, time.Since(start))
cached := cacheData.(ThumbCache)

2
internal/api/folder.go
View file

@ -56,7 +56,7 @@ func GetFolders(router *gin.RouterGroup, urlPath, rootName, rootPath string) {
if cacheData, ok := cache.Get(cacheKey); ok {
cached := cacheData.(FoldersResponse)
log.Debugf("cache hit for %s [%s]", cacheKey, time.Since(start))
log.Debugf("api: cache hit for %s [%s]", cacheKey, time.Since(start))
c.JSON(http.StatusOK, cached)
return

2
internal/api/photo_thumb.go
View file

@ -55,7 +55,7 @@ func GetThumb(router *gin.RouterGroup) {
cacheKey := CacheKey("thumbs", fileHash, typeName)
if cacheData, ok := cache.Get(cacheKey); ok {
log.Debugf("cache hit for %s [%s]", cacheKey, time.Since(start))
log.Debugf("api: cache hit for %s [%s]", cacheKey, time.Since(start))
cached := cacheData.(ThumbCache)

6
internal/commands/cleanup.go
View file

@ -13,7 +13,7 @@ import (
// CleanUpCommand registers the cleanup command.
var CleanUpCommand = cli.Command{
Name: "cleanup",
Usage: "Removes orphaned thumbnails and index entries",
Usage: "Removes orphan index entries and thumbnails",
Flags: cleanUpFlags,
Action: cleanUpAction,
}
@ -25,7 +25,7 @@ var cleanUpFlags = []cli.Flag{
},
}
// cleanUpAction removes orphaned thumbnails and index entries.
// cleanUpAction removes orphan index entries and thumbnails.
func cleanUpAction(ctx *cli.Context) error {
start := time.Now()
@ -56,7 +56,7 @@ func cleanUpAction(ctx *cli.Context) error {
} else {
elapsed := time.Since(start)
log.Infof("cleanup: removed %d orphaned thumbnails and %d photos in %s", thumbs, orphans, elapsed)
log.Infof("cleanup: removed %d index entries and %d orphan thumbnails in %s", orphans, thumbs, elapsed)
}
conf.Shutdown()

4
internal/commands/index.go
View file

@ -28,7 +28,7 @@ var indexFlags = []cli.Flag{
},
cli.BoolFlag{
Name: "cleanup",
Usage: "removes orphaned thumbnails and index entries",
Usage: "removes orphan index entries and thumbnails",
},
}
@ -95,7 +95,7 @@ func indexAction(ctx *cli.Context) error {
if thumbs, orphans, err := cleanUp.Start(opt); err != nil {
return err
} else {
log.Infof("cleanup: removed %d orphaned thumbnails and %d photos", thumbs, orphans)
log.Infof("cleanup: removed %d index entries and %d orphan thumbnails", orphans, thumbs)
}
}

106
internal/entity/camera.go
View file

@ -34,102 +34,6 @@ var UnknownCamera = Camera{
CameraModel: "Unknown",
}
var CameraMakes = map[string]string{
"OLYMPUS": "Olympus",
"OLYMPUS CORPORATION": "Olympus",
"OLYMPUS DIGITAL CAMERA": "Olympus",
"OLYMPUS IMAGING CORP.": "Olympus",
"OLYMPUS OPTICAL CO.,LTD": "Olympus",
"samsung": "Samsung",
}
var CameraModels = map[string]string{
"iPhone SE (1st generation)": "iPhone SE",
"iPhone SE (2nd generation)": "iPhone SE",
"iPhone SE (3rd generation)": "iPhone SE",
"WAS-LX1": "P10 lite",
"WAS-LX2": "P10 lite",
"WAS-LX3": "P10 lite",
"WAS-LX1A": "P10 lite",
"WAS-LX2J": "P10 lite",
"WAS-L03T": "P10 lite",
"WAS-AL00": "P10 lite",
"WAS-TL10": "P10 lite",
"VTR-L29": "P10",
"VTR-AL00": "P10",
"VTR-TL00": "P10",
"VTR-L09": "P10",
"EML-AL00": "P20",
"EML-L09": "P20",
"EML-L09C": "P20",
"EML-L29": "P20",
"EML-L29C": "P20",
"CLT-AL00": "P20 Pro",
"CLT-AL01": "P20 Pro",
"CLT-TL01": "P20 Pro",
"CLT-L09": "P20 Pro",
"CLT-L29": "P20 Pro",
"ELE-L29": "P30",
"ELE-AL00": "P30",
"ELE-L04": "P30",
"ELE-L09": "P30",
"ELE-TL00": "P30",
"VOG-L29": "P30 Pro",
"VOG-L09": "P30 Pro",
"VOG-L04": "P30 Pro",
"VOG-AL00": "P30 Pro",
"VOG-AL10": "P30 Pro",
"VOG-TL00": "P30 Pro",
"MAR-L01A": "P30 lite",
"MAR-L21A": "P30 lite",
"MAR-LX1A": "P30 lite",
"MAR-LX1M": "P30 lite",
"MAR-LX2": "P30 lite",
"MAR-L21MEA": "P30 lite",
"MAR-L22A": "P30 lite",
"MAR-L22B": "P30 lite",
"MAR-LX3A": "P30 lite",
"ANA-AN00": "P40",
"ANA-TN00": "P40",
"ELS-AN00": "P40 Pro",
"ELS-TN00": "P40 Pro",
"ELS-NX9": "P40 Pro",
"ELS-N04": "P40 Pro",
"JNY-L21A": "P40 lite",
"JNY-L01A": "P40 lite",
"JNY-L21B": "P40 lite",
"JNY-L22A": "P40 lite",
"JNY-L02A": "P40 lite",
"JNY-L22B": "P40 lite",
"STK-LX1": "Honor 9X",
"HLK-AL00": "Honor 9X",
"HLK-TL00": "Honor 9X",
"SNE-AL00": "Mate 20 lite",
"SNE-LX1": "Mate 20 lite",
"SNE-LX2": "Mate 20 lite",
"SNE-LX3": "Mate 20 lite",
"INE-LX2": "Mate 20 lite",
"HMA-L29": "Mate 20",
"HMA-L09": "Mate 20",
"HMA-LX9": "Mate 20",
"HMA-AL00": "Mate 20",
"HMA-TL00": "Mate 20",
"LYA-L09": "Mate 20 Pro",
"LYA-L29": "Mate 20 Pro",
"LYA-AL00": "Mate 20 Pro",
"LYA-AL10": "Mate 20 Pro",
"LYA-TL00": "Mate 20 Pro",
"LYA-L0C": "Mate 20 Pro",
"TAS-L09": "Mate 30",
"TAS-L29": "Mate 30",
"TAS-AL00": "Mate 30",
"TAS-TL00": "Mate 30",
"LIO-L09": "Mate 30 Pro",
"LIO-L29": "Mate 30 Pro",
"LIO-AL00": "Mate 30 Pro",
"LIO-TL00": "Mate 30 Pro",
}
// CreateUnknownCamera initializes the database with an unknown camera if not exists
func CreateUnknownCamera() {
FirstOrCreateCamera(&UnknownCamera)
@ -191,9 +95,16 @@ func FirstOrCreateCamera(m *Camera) *Camera {
return &UnknownCamera
}
if cacheData, ok := cameraCache.Get(m.CameraSlug); ok {
log.Debugf("camera: cache hit for %s", m.CameraSlug)
return cacheData.(*Camera)
}
result := Camera{}
if res := Db().Where("camera_slug = ?", m.CameraSlug).First(&result); res.Error == nil {
cameraCache.SetDefault(m.CameraSlug, &result)
return &result
} else if err := m.Create(); err == nil {
if !m.Unknown() {
@ -204,8 +115,11 @@ func FirstOrCreateCamera(m *Camera) *Camera {
})
}
cameraCache.SetDefault(m.CameraSlug, m)
return m
} else if res := Db().Where("camera_slug = ?", m.CameraSlug).First(&result); res.Error == nil {
cameraCache.SetDefault(m.CameraSlug, &result)
return &result
} else {
log.Errorf("camera: %s (create %s)", err.Error(), txt.Quote(m.String()))

13
internal/entity/camera_cache.go Normal file
View file

@ -0,0 +1,13 @@
package entity
import (
"time"
gc "github.com/patrickmn/go-cache"
)
var cameraCache = gc.New(time.Hour, 15*time.Minute)
func FlushCameraCache() {
cameraCache.Flush()
}

97
internal/entity/camera_makes.go Normal file
View file

@ -0,0 +1,97 @@
package entity
var CameraMakes = map[string]string{
"OLYMPUS": "Olympus",
"OLYMPUS CORPORATION": "Olympus",
"OLYMPUS DIGITAL CAMERA": "Olympus",
"OLYMPUS IMAGING CORP.": "Olympus",
"OLYMPUS OPTICAL CO.,LTD": "Olympus",
"samsung": "Samsung",
}
var CameraModels = map[string]string{
"iPhone SE (1st generation)": "iPhone SE",
"iPhone SE (2nd generation)": "iPhone SE",
"iPhone SE (3rd generation)": "iPhone SE",
"WAS-LX1": "P10 lite",
"WAS-LX2": "P10 lite",
"WAS-LX3": "P10 lite",
"WAS-LX1A": "P10 lite",
"WAS-LX2J": "P10 lite",
"WAS-L03T": "P10 lite",
"WAS-AL00": "P10 lite",
"WAS-TL10": "P10 lite",
"VTR-L29": "P10",
"VTR-AL00": "P10",
"VTR-TL00": "P10",
"VTR-L09": "P10",
"EML-AL00": "P20",
"EML-L09": "P20",
"EML-L09C": "P20",
"EML-L29": "P20",
"EML-L29C": "P20",
"CLT-AL00": "P20 Pro",
"CLT-AL01": "P20 Pro",
"CLT-TL01": "P20 Pro",
"CLT-L09": "P20 Pro",
"CLT-L29": "P20 Pro",
"ELE-L29": "P30",
"ELE-AL00": "P30",
"ELE-L04": "P30",
"ELE-L09": "P30",
"ELE-TL00": "P30",
"VOG-L29": "P30 Pro",
"VOG-L09": "P30 Pro",
"VOG-L04": "P30 Pro",
"VOG-AL00": "P30 Pro",
"VOG-AL10": "P30 Pro",
"VOG-TL00": "P30 Pro",
"MAR-L01A": "P30 lite",
"MAR-L21A": "P30 lite",
"MAR-LX1A": "P30 lite",
"MAR-LX1M": "P30 lite",
"MAR-LX2": "P30 lite",
"MAR-L21MEA": "P30 lite",
"MAR-L22A": "P30 lite",
"MAR-L22B": "P30 lite",
"MAR-LX3A": "P30 lite",
"ANA-AN00": "P40",
"ANA-TN00": "P40",
"ELS-AN00": "P40 Pro",
"ELS-TN00": "P40 Pro",
"ELS-NX9": "P40 Pro",
"ELS-N04": "P40 Pro",
"JNY-L21A": "P40 lite",
"JNY-L01A": "P40 lite",
"JNY-L21B": "P40 lite",
"JNY-L22A": "P40 lite",
"JNY-L02A": "P40 lite",
"JNY-L22B": "P40 lite",
"STK-LX1": "Honor 9X",
"HLK-AL00": "Honor 9X",
"HLK-TL00": "Honor 9X",
"SNE-AL00": "Mate 20 lite",
"SNE-LX1": "Mate 20 lite",
"SNE-LX2": "Mate 20 lite",
"SNE-LX3": "Mate 20 lite",
"INE-LX2": "Mate 20 lite",
"HMA-L29": "Mate 20",
"HMA-L09": "Mate 20",
"HMA-LX9": "Mate 20",
"HMA-AL00": "Mate 20",
"HMA-TL00": "Mate 20",
"LYA-L09": "Mate 20 Pro",
"LYA-L29": "Mate 20 Pro",
"LYA-AL00": "Mate 20 Pro",
"LYA-AL10": "Mate 20 Pro",
"LYA-TL00": "Mate 20 Pro",
"LYA-L0C": "Mate 20 Pro",
"TAS-L09": "Mate 30",
"TAS-L29": "Mate 30",
"TAS-AL00": "Mate 30",
"TAS-TL00": "Mate 30",
"LIO-L09": "Mate 30 Pro",
"LIO-L29": "Mate 30 Pro",
"LIO-AL00": "Mate 30 Pro",
"LIO-TL00": "Mate 30 Pro",
}

12
internal/entity/country.go
View file

@ -66,9 +66,16 @@ func (m *Country) Create() error {
// FirstOrCreateCountry returns the existing row, inserts a new row or nil in case of errors.
func FirstOrCreateCountry(m *Country) *Country {
if cacheData, ok := countryCache.Get(m.ID); ok {
log.Debugf("country: cache hit for %s", m.ID)
return cacheData.(*Country)
}
result := Country{}
if findErr := Db().Where("id = ?", m.ID).First(&result).Error; findErr == nil {
countryCache.SetDefault(m.ID, &result)
return &result
} else if createErr := m.Create(); createErr == nil {
if !m.Unknown() {
@ -78,15 +85,16 @@ func FirstOrCreateCountry(m *Country) *Country {
"count": 1,
})
}
countryCache.SetDefault(m.ID, m)
return m
} else if err := Db().Where("id = ?", m.ID).First(&result).Error; err == nil {
countryCache.SetDefault(m.ID, &result)
return &result
} else {
log.Errorf("country: %s (find or create %s)", createErr, m.ID)
}
return nil
return &UnknownCountry
}
// AfterCreate sets the New column used for database callback

13
internal/entity/country_cache.go Normal file
View file

@ -0,0 +1,13 @@
package entity
import (
"time"
gc "github.com/patrickmn/go-cache"
)
var countryCache = gc.New(time.Hour, 15*time.Minute)
func FlushCountryCache() {
countryCache.Flush()
}

10
internal/entity/lens.go
View file

@ -96,9 +96,16 @@ func FirstOrCreateLens(m *Lens) *Lens {
return &UnknownLens
}
if cacheData, ok := lensCache.Get(m.LensSlug); ok {
log.Debugf("lens: cache hit for %s", m.LensSlug)
return cacheData.(*Lens)
}
result := Lens{}
if res := Db().Where("lens_slug = ?", m.LensSlug).First(&result); res.Error == nil {
lensCache.SetDefault(m.LensSlug, &result)
return &result
} else if err := m.Create(); err == nil {
if !m.Unknown() {
@ -109,8 +116,11 @@ func FirstOrCreateLens(m *Lens) *Lens {
})
}
lensCache.SetDefault(m.LensSlug, m)
return m
} else if res := Db().Where("lens_slug = ?", m.LensSlug).First(&result); res.Error == nil {
lensCache.SetDefault(m.LensSlug, &result)
return &result
} else {
log.Errorf("lens: %s (create %s)", err.Error(), txt.Quote(m.String()))

13
internal/entity/lens_cache.go Normal file
View file

@ -0,0 +1,13 @@
package entity
import (
"time"
gc "github.com/patrickmn/go-cache"
)
var lensCache = gc.New(time.Hour, 15*time.Minute)
func FlushLensCache() {
lensCache.Flush()
}

72
internal/entity/photo.go
View file

@ -238,7 +238,7 @@ func (m *Photo) Save() error {
photoMutex.Lock()
defer photoMutex.Unlock()
if err := Save(m, "ID"); err != nil {
if err := Save(m, "ID", "PhotoUID"); err != nil {
return err
}
@ -606,6 +606,16 @@ func (m *Photo) NoCameraSerial() bool {
return m.CameraSerial == ""
}
// UnknownCamera test if the camera is unknown.
func (m *Photo) UnknownCamera() bool {
return m.CameraID == 0 || m.CameraID == UnknownCamera.ID
}
// UnknownLens test if the lens is unknown.
func (m *Photo) UnknownLens() bool {
return m.LensID == 0 || m.LensID == UnknownLens.ID
}
// HasTitle checks if the photo has a title.
func (m *Photo) HasTitle() bool {
return m.PhotoTitle != ""
@ -940,6 +950,66 @@ func (m *Photo) SetCoordinates(lat, lng float32, altitude int, source string) {
m.PlaceSrc = source
}
// SetCamera updates the camera.
func (m *Photo) SetCamera(camera *Camera, source string) {
if camera == nil {
log.Warnf("photo: failed updating camera from source '%s'", source)
return
}
if camera.Unknown() {
return
}
if SrcPriority[source] < SrcPriority[m.CameraSrc] && !m.UnknownCamera() {
return
}
m.CameraID = camera.ID
m.Camera = camera
m.CameraSrc = source
}
// SetLens updates the lens.
func (m *Photo) SetLens(lens *Lens, source string) {
if lens == nil {
log.Warnf("photo: failed updating lens from source '%s'", source)
return
}
if lens.Unknown() {
return
}
if SrcPriority[source] < SrcPriority[m.CameraSrc] && !m.UnknownLens() {
return
}
m.LensID = lens.ID
m.Lens = lens
}
// SetExposure updates the photo exposure details.
func (m *Photo) SetExposure(focalLength int, fNumber float32, iso int, exposure, source string) {
hasPriority := SrcPriority[source] >= SrcPriority[m.CameraSrc]
if focalLength > 0 && (hasPriority || m.PhotoFocalLength <= 0) {
m.PhotoFocalLength = focalLength
}
if fNumber > 0 && (hasPriority || m.PhotoFNumber <= 0) {
m.PhotoFNumber = fNumber
}
if iso > 0 && (hasPriority || m.PhotoIso <= 0) {
m.PhotoIso = iso
}
if exposure != "" && (hasPriority || m.PhotoExposure == "") {
m.PhotoExposure = exposure
}
}
// AllFilesMissing returns true, if all files for this photo are missing.
func (m *Photo) AllFilesMissing() bool {
count := 0

14
internal/entity/photo_test.go
View file

@ -17,7 +17,7 @@ func TestSavePhotoForm(t *testing.T) {
TakenSrc: "manual",
TimeZone: "test",
PhotoTitle: "Pink beach",
TitleSrc: "manual",
TitleSrc: SrcManual,
PhotoFavorite: true,
PhotoPrivate: true,
PhotoType: "image",
@ -29,10 +29,10 @@ func TestSavePhotoForm(t *testing.T) {
PhotoFNumber: 3.3,
PhotoExposure: "exposure",
CameraID: uint(3),
CameraSrc: "meta",
CameraSrc: SrcMeta,
LensID: uint(6),
CellID: "1234",
PlaceSrc: "manual",
PlaceSrc: SrcManual,
PlaceID: "765",
PhotoCountry: "de",
Details: form.Details{
@ -46,11 +46,9 @@ func TestSavePhotoForm(t *testing.T) {
},
}
m := PhotoFixtures["Photo08"]
m := PhotoFixtures.Get("Photo08")
err := SavePhotoForm(m, f)
if err != nil {
if err := SavePhotoForm(m, f); err != nil {
t.Fatal(err)
}
@ -864,7 +862,7 @@ func TestPhoto_Save(t *testing.T) {
func TestPhoto_Find(t *testing.T) {
t.Run("success", func(t *testing.T) {
photo := Photo{PhotoUID: "567", ID: 55, PhotoName: "Holiday", OriginalName: "holidayOriginal2"}
photo := Photo{PhotoUID: "pt9atdre2lvl0yhx", PhotoName: "Holiday", OriginalName: "holidayOriginal2"}
err := photo.Save()
if err != nil {
t.Fatal(err)

41
internal/entity/save.go
View file

@ -6,11 +6,42 @@ import (
"strings"
)
// Values returns entity values as string map.
func Values(m interface{}, omit ...string) (result map[string]interface{}) {
skip := func(name string) bool {
for _, s := range omit {
if name == s {
return true
}
}
return false
}
result = make(map[string]interface{})
elem := reflect.ValueOf(m).Elem()
relType := elem.Type()
for i := 0; i < relType.NumField(); i++ {
name := relType.Field(i).Name
if skip(name) {
continue
}
result[name] = elem.Field(i).Interface()
}
return result
}
// Save updates an entity in the database, or inserts if it doesn't exist.
func Save(m interface{}, primaryKeys ...string) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("save: %s (panic)", r)
log.Error(err)
}
}()
@ -32,6 +63,7 @@ func Update(m interface{}, primaryKeys ...string) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("update: %s (panic)", r)
log.Error(err)
}
}()
@ -45,12 +77,13 @@ func Update(m interface{}, primaryKeys ...string) (err error) {
}
// Update all values except primary keys.
if res := UnscopedDb().Model(m).Select("*").Omit(primaryKeys...).Updates(m); res.Error != nil {
if res := UnscopedDb().Model(m).Updates(Values(m, primaryKeys...)); res.Error != nil {
return res.Error
} else if res.RowsAffected == 0 {
return fmt.Errorf("no entity found for updating")
} else if res.RowsAffected > 1 {
log.Warnf("update: more than one row affected - bug?")
log.Warnf("update: more than one row affected")
} else if res.RowsAffected == 0 {
// MariaDB may report zero rows in case no data was actually changed, even though the row exists.
log.Tracef("update: no rows affected")
}
return nil

22
internal/photoprism/cleanup.go
View file

@ -33,7 +33,7 @@ func NewCleanUp(conf *config.Config) *CleanUp {
return instance
}
// Start removes orphaned thumbnails and index entries.
// Start removes orphan index entries and thumbnails.
func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error) {
defer func() {
if r := recover(); r != nil {
@ -53,7 +53,7 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
log.Infof("cleanup: dry run, nothing will actually be removed")
}
// Find and remove orphaned thumbnail thumbs.
// Find and remove orphan thumbnail files.
hashes, err := query.FileHashes()
if err != nil {
@ -82,12 +82,12 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
// Do nothing.
} else if opt.Dry {
thumbs++
log.Debugf("cleanup: orphaned thumbnail %s would be removed", logName)
log.Debugf("cleanup: orphan thumbnail %s would be removed", logName)
} else if err := os.Remove(fileName); err != nil {
log.Warnf("cleanup: %s in %s", err, logName)
} else {
thumbs++
log.Debugf("cleanup: removed orphaned thumbnail %s", logName)
log.Debugf("cleanup: removed orphan thumbnail %s", logName)
}
return nil
@ -95,8 +95,8 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
return thumbs, orphans, err
}
// Find and remove orphaned photo index entries without thumbs.
photos, err := query.PhotosOrphaned()
// Find and remove orphan photo index entries.
photos, err := query.OrphanPhotos()
if err != nil {
return thumbs, orphans, err
@ -111,19 +111,23 @@ func (w *CleanUp) Start(opt CleanUpOptions) (thumbs int, orphans int, err error)
if opt.Dry {
orphans++
log.Infof("cleanup: orphaned photo %s would be removed", txt.Quote(p.PhotoUID))
log.Infof("cleanup: orphan photo %s would be removed", txt.Quote(p.PhotoUID))
continue
}
if err := Delete(p); err != nil {
log.Errorf("cleanup: %s (remove orphan)", err.Error())
log.Errorf("cleanup: %s (remove orphan photo)", err.Error())
} else {
orphans++
deleted = append(deleted, p.PhotoUID)
log.Debugf("cleanup: removed orphaned photo %s", p.PhotoUID)
log.Debugf("cleanup: removed orphan photo %s", p.PhotoUID)
}
}
if err := query.PurgeOrphans(); err != nil {
log.Errorf("cleanup: %s (purge orphans)", err)
}
// Update counts and views if needed.
if len(deleted) > 0 {
if err := entity.UpdatePhotoCounts(); err != nil {

2
internal/photoprism/files.go
View file

@ -35,7 +35,7 @@ func (m *Files) Init() error {
return nil
}
if err := query.PurgeDuplicates(); err != nil {
if err := query.PurgeOrphanDuplicates(); err != nil {
return fmt.Errorf("%s (purge duplicates)", err.Error())
}

90
internal/photoprism/index_mediafile.go
View file

@ -385,33 +385,9 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
photo.PhotoResolution = res
}
if photo.CameraSrc == entity.SrcAuto {
// Set UpdateCamera, Lens, Focal Length and F Number.
photo.Camera = entity.FirstOrCreateCamera(entity.NewCamera(m.CameraModel(), m.CameraMake()))
if photo.Camera != nil {
photo.CameraID = photo.Camera.ID
} else {
photo.CameraID = entity.UnknownCamera.ID
}
if photo.CameraID != entity.UnknownCamera.ID {
photo.CameraSrc = entity.SrcMeta
}
photo.Lens = entity.FirstOrCreateLens(entity.NewLens(m.LensModel(), m.LensMake()))
if photo.Lens != nil {
photo.LensID = photo.Lens.ID
} else {
photo.LensID = entity.UnknownLens.ID
}
photo.PhotoFocalLength = m.FocalLength()
photo.PhotoFNumber = m.FNumber()
photo.PhotoIso = m.Iso()
photo.PhotoExposure = m.Exposure()
}
photo.SetCamera(entity.FirstOrCreateCamera(entity.NewCamera(m.CameraModel(), m.CameraMake())), entity.SrcMeta)
photo.SetLens(entity.FirstOrCreateLens(entity.NewLens(m.LensModel(), m.LensMake())), entity.SrcMeta)
photo.SetExposure(m.FocalLength(), m.FNumber(), m.Iso(), m.Exposure(), entity.SrcMeta)
}
if photo.TypeSrc == entity.SrcAuto {
@ -459,33 +435,9 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
photo.PhotoResolution = res
}
if photo.CameraSrc == entity.SrcAuto {
// Set UpdateCamera, Lens, Focal Length and F Number.
photo.Camera = entity.FirstOrCreateCamera(entity.NewCamera(m.CameraModel(), m.CameraMake()))
if photo.Camera != nil {
photo.CameraID = photo.Camera.ID
} else {
photo.CameraID = entity.UnknownCamera.ID
}
if photo.CameraID != entity.UnknownCamera.ID {
photo.CameraSrc = entity.SrcMeta
}
photo.Lens = entity.FirstOrCreateLens(entity.NewLens(m.LensModel(), m.LensMake()))
if photo.Lens != nil {
photo.LensID = photo.Lens.ID
} else {
photo.LensID = entity.UnknownLens.ID
}
photo.PhotoFocalLength = m.FocalLength()
photo.PhotoFNumber = m.FNumber()
photo.PhotoIso = m.Iso()
photo.PhotoExposure = m.Exposure()
}
photo.SetCamera(entity.FirstOrCreateCamera(entity.NewCamera(m.CameraModel(), m.CameraMake())), entity.SrcMeta)
photo.SetLens(entity.FirstOrCreateLens(entity.NewLens(m.LensModel(), m.LensMake())), entity.SrcMeta)
photo.SetExposure(m.FocalLength(), m.FNumber(), m.Iso(), m.Exposure(), entity.SrcMeta)
}
if photo.TypeSrc == entity.SrcAuto {
@ -565,33 +517,9 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
}
}
if photo.CameraSrc == entity.SrcAuto {
// Set UpdateCamera, Lens, Focal Length and F Number.
photo.Camera = entity.FirstOrCreateCamera(entity.NewCamera(m.CameraModel(), m.CameraMake()))
if photo.Camera != nil {
photo.CameraID = photo.Camera.ID
} else {
photo.CameraID = entity.UnknownCamera.ID
}
if photo.CameraID != entity.UnknownCamera.ID {
photo.CameraSrc = entity.SrcMeta
}
photo.Lens = entity.FirstOrCreateLens(entity.NewLens(m.LensModel(), m.LensMake()))
if photo.Lens != nil {
photo.LensID = photo.Lens.ID
} else {
photo.LensID = entity.UnknownLens.ID
}
photo.PhotoFocalLength = m.FocalLength()
photo.PhotoFNumber = m.FNumber()
photo.PhotoIso = m.Iso()
photo.PhotoExposure = m.Exposure()
}
photo.SetCamera(entity.FirstOrCreateCamera(entity.NewCamera(m.CameraModel(), m.CameraMake())), entity.SrcMeta)
photo.SetLens(entity.FirstOrCreateLens(entity.NewLens(m.LensModel(), m.LensMake())), entity.SrcMeta)
photo.SetExposure(m.FocalLength(), m.FNumber(), m.Iso(), m.Exposure(), entity.SrcMeta)
var locLabels classify.Labels

8
internal/photoprism/purge.go
View file

@ -251,12 +251,8 @@ func (w *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPhot
return purgedFiles, purgedPhotos, err
}
if err := query.PurgeDuplicates(); err != nil {
log.Errorf("purge: %s (duplicates)", err)
}
if err := query.PurgeUnusedCountries(); err != nil {
log.Errorf("purge: %s (countries)", err)
if err := query.PurgeOrphans(); err != nil {
log.Errorf("purge: %s (orphans)", err)
}
if err := query.UpdateMissingAlbumEntries(); err != nil {

9
internal/query/countries.go
View file

@ -1,9 +0,0 @@
package query
// PurgeUnusedCountries removes countries without any photos.
func PurgeUnusedCountries() error {
switch DbDialect() {
default:
return UnscopedDb().Exec(`DELETE FROM countries WHERE id NOT IN (SELECT photo_country FROM photos)`).Error
}
}

11
internal/query/countries_test.go
View file

@ -1,11 +0,0 @@
package query
import (
"testing"
)
func TestPurgeUnusedCountries(t *testing.T) {
if err := PurgeUnusedCountries(); err != nil {
t.Fatal(err)
}
}

10
internal/query/duplicates.go
View file

@ -6,16 +6,6 @@ import (
"github.com/photoprism/photoprism/internal/entity"
)
// PurgeDuplicates deletes all files from the duplicates table that don't exist in the files table.
func PurgeDuplicates() error {
if err := UnscopedDb().Delete(entity.Duplicate{}, "file_hash IN (SELECT d.file_hash FROM duplicates d LEFT JOIN files f ON d.file_hash = f.file_hash AND f.file_missing = 0 AND f.deleted_at IS NULL WHERE f.file_hash IS NULL)").Error; err == nil {
return nil
}
// MySQL fallback, see https://github.com/photoprism/photoprism/issues/599
return UnscopedDb().Delete(entity.Duplicate{}, "file_hash IN (SELECT file_hash FROM (SELECT d.file_hash FROM duplicates d LEFT JOIN files f ON d.file_hash = f.file_hash AND f.file_missing = 0 AND f.deleted_at IS NULL WHERE f.file_hash IS NULL) AS tmp)").Error
}
// Duplicates finds duplicate files in the range of limit and offset sorted by file name.
func Duplicates(limit, offset int, pathName string) (files entity.Duplicates, err error) {
if strings.HasPrefix(pathName, "/") {

34
internal/query/duplicates_test.go
View file

@ -2,38 +2,12 @@ package query
import (
"testing"
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/stretchr/testify/assert"
)
func TestPurgeDuplicates(t *testing.T) {
fileName := "hd89e5yhb8p9h.jpg"
if err := entity.AddDuplicate(
fileName,
entity.RootOriginals,
"2cad9168fa6acc5c5c2965ddf6ec465ca42fd811",
661858,
time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
); err != nil {
func TestDuplicates(t *testing.T) {
if files, err := Duplicates(10, 0, ""); err != nil {
t.Fatal(err)
}
d := &entity.Duplicate{FileName: fileName, FileRoot: entity.RootOriginals}
if err := d.Find(); err != nil {
t.Fatal(err)
}
err := PurgeDuplicates()
assert.NoError(t, err)
dp := &entity.Duplicate{FileName: fileName, FileRoot: entity.RootOriginals}
if err := dp.Find(); err == nil {
t.Fatalf("duplicate should be removed: %+v", dp)
} else if files == nil {
t.Fatal("files must not be nil")
}
}

4
internal/query/photo.go
View file

@ -118,8 +118,8 @@ func PhotosCheck(limit, offset int, delay time.Duration) (entities entity.Photos
return entities, err
}
// PhotosOrphaned finds orphaned index entries that may be removed.
func PhotosOrphaned() (photos entity.Photos, err error) {
// OrphanPhotos finds orphan index entries that may be removed.
func OrphanPhotos() (photos entity.Photos, err error) {
err = UnscopedDb().
Raw(`SELECT * FROM photos WHERE
deleted_at IS NOT NULL

4
internal/query/photo_test.go
View file

@ -80,8 +80,8 @@ func TestPhotosCheck(t *testing.T) {
assert.IsType(t, entity.Photos{}, result)
}
func TestPhotosOrphaned(t *testing.T) {
result, err := PhotosOrphaned()
func TestOrphanPhotos(t *testing.T) {
result, err := OrphanPhotos()
if err != nil {
t.Fatal(err)

58
internal/query/purge.go Normal file
View file

@ -0,0 +1,58 @@
package query
import "github.com/photoprism/photoprism/internal/entity"
// PurgeOrphans removes orphan database entries.
func PurgeOrphans() error {
if err := PurgeOrphanDuplicates(); err != nil {
return err
}
if err := PurgeOrphanCountries(); err != nil {
return err
}
if err := PurgeOrphanCameras(); err != nil {
return err
}
if err := PurgeOrphanLenses(); err != nil {
return err
}
return nil
}
// PurgeOrphanDuplicates deletes all files from the duplicates table that don't exist in the files table.
func PurgeOrphanDuplicates() error {
if err := UnscopedDb().Delete(entity.Duplicate{}, "file_hash IN (SELECT d.file_hash FROM duplicates d LEFT JOIN files f ON d.file_hash = f.file_hash AND f.file_missing = 0 AND f.deleted_at IS NULL WHERE f.file_hash IS NULL)").Error; err == nil {
return nil
}
// MySQL fallback, see https://github.com/photoprism/photoprism/issues/599
return UnscopedDb().Delete(entity.Duplicate{}, "file_hash IN (SELECT file_hash FROM (SELECT d.file_hash FROM duplicates d LEFT JOIN files f ON d.file_hash = f.file_hash AND f.file_missing = 0 AND f.deleted_at IS NULL WHERE f.file_hash IS NULL) AS tmp)").Error
}
// PurgeOrphanCountries removes countries without any photos.
func PurgeOrphanCountries() error {
entity.FlushCountryCache()
switch DbDialect() {
default:
return UnscopedDb().Exec(`DELETE FROM countries WHERE country_slug <> ? AND id NOT IN (SELECT photo_country FROM photos)`, entity.UnknownCountry.CountrySlug).Error
}
}
// PurgeOrphanCameras removes cameras without any photos.
func PurgeOrphanCameras() error {
entity.FlushCameraCache()
switch DbDialect() {
default:
return UnscopedDb().Exec(`DELETE FROM cameras WHERE camera_slug <> ? AND id NOT IN (SELECT camera_id FROM photos)`, entity.UnknownCamera.CameraSlug).Error
}
}
// PurgeOrphanLenses removes cameras without any photos.
func PurgeOrphanLenses() error {
entity.FlushLensCache()
switch DbDialect() {
default:
return UnscopedDb().Exec(`DELETE FROM lenses WHERE lens_slug <> ? AND id NOT IN (SELECT lens_id FROM photos)`, entity.UnknownLens.LensSlug).Error
}
}

75
internal/query/purge_test.go Normal file
View file

@ -0,0 +1,75 @@
package query
import (
"testing"
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/stretchr/testify/assert"
)
func TestPurgeOrphans(t *testing.T) {
fileName := "hd89e5yhb8p9h.jpg"
if err := entity.AddDuplicate(
fileName,
entity.RootOriginals,
"2cad9168fa6acc5c5c2965ddf6ec465ca42fd811",
661858,
time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
); err != nil {
t.Fatal(err)
}
if err := PurgeOrphans(); err != nil {
t.Fatal(err)
}
}
func TestPurgeFileDuplicates(t *testing.T) {
fileName := "hd89e5yhb8p9h.jpg"
if err := entity.AddDuplicate(
fileName,
entity.RootOriginals,
"2cad9168fa6acc5c5c2965ddf6ec465ca42fd811",
661858,
time.Date(2019, 3, 6, 2, 6, 51, 0, time.UTC).Unix(),
); err != nil {
t.Fatal(err)
}
d := &entity.Duplicate{FileName: fileName, FileRoot: entity.RootOriginals}
if err := d.Find(); err != nil {
t.Fatal(err)
}
err := PurgeOrphanDuplicates()
assert.NoError(t, err)
dp := &entity.Duplicate{FileName: fileName, FileRoot: entity.RootOriginals}
if err := dp.Find(); err == nil {
t.Fatalf("duplicate should be removed: %+v", dp)
}
}
func TestPurgeUnusedCountries(t *testing.T) {
if err := PurgeOrphanCountries(); err != nil {
t.Fatal(err)
}
}
func TestPurgeUnusedCameras(t *testing.T) {
if err := PurgeOrphanCameras(); err != nil {
t.Fatal(err)
}
}
func TestPurgeUnusedLenses(t *testing.T) {
if err := PurgeOrphanLenses(); err != nil {
t.Fatal(err)
}
}