diff --git a/docs/docs/guides/database-queries.md b/docs/docs/guides/database-queries.md index e8be92ef4..04eaf81f7 100644 --- a/docs/docs/guides/database-queries.md +++ b/docs/docs/guides/database-queries.md @@ -66,6 +66,10 @@ ORDER BY "users"."email"; ``` +```sql title="Failed file movements" +SELECT * FROM "move_history"; +``` + ## Users ```sql title="List" diff --git a/server/src/domain/asset/asset.service.spec.ts b/server/src/domain/asset/asset.service.spec.ts index d2c0f7f3b..20e86f159 100644 --- a/server/src/domain/asset/asset.service.spec.ts +++ b/server/src/domain/asset/asset.service.spec.ts @@ -10,6 +10,8 @@ import { newCommunicationRepositoryMock, newCryptoRepositoryMock, newJobRepositoryMock, + newMoveRepositoryMock, + newPersonRepositoryMock, newStorageRepositoryMock, newSystemConfigRepositoryMock, } from '@test'; @@ -22,6 +24,8 @@ import { ICommunicationRepository, ICryptoRepository, IJobRepository, + IMoveRepository, + IPersonRepository, IStorageRepository, ISystemConfigRepository, JobItem, @@ -160,6 +164,8 @@ describe(AssetService.name, () => { let assetMock: jest.Mocked; let cryptoMock: jest.Mocked; let jobMock: jest.Mocked; + let moveMock: jest.Mocked; + let personMock: jest.Mocked; let storageMock: jest.Mocked; let communicationMock: jest.Mocked; let configMock: jest.Mocked; @@ -174,9 +180,21 @@ describe(AssetService.name, () => { communicationMock = newCommunicationRepositoryMock(); cryptoMock = newCryptoRepositoryMock(); jobMock = newJobRepositoryMock(); + moveMock = newMoveRepositoryMock(); + personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); configMock = newSystemConfigRepositoryMock(); - sut = new AssetService(accessMock, assetMock, cryptoMock, jobMock, configMock, storageMock, communicationMock); + sut = new AssetService( + accessMock, + assetMock, + cryptoMock, + jobMock, + configMock, + moveMock, + personMock, + storageMock, + communicationMock, + ); when(assetMock.getById) .calledWith(assetStub.livePhotoStillAsset.id) diff --git a/server/src/domain/asset/asset.service.ts b/server/src/domain/asset/asset.service.ts index d809a9cb1..15a1c67bc 100644 --- a/server/src/domain/asset/asset.service.ts +++ b/server/src/domain/asset/asset.service.ts @@ -16,6 +16,8 @@ import { ICommunicationRepository, ICryptoRepository, IJobRepository, + IMoveRepository, + IPersonRepository, IStorageRepository, ISystemConfigRepository, ImmichReadStream, @@ -80,12 +82,14 @@ export class AssetService { @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, + @Inject(IMoveRepository) moveRepository: IMoveRepository, + @Inject(IPersonRepository) personRepository: IPersonRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository, ) { this.access = new AccessCore(accessRepository); - this.storageCore = new StorageCore(storageRepository); this.configCore = SystemConfigCore.create(configRepository); + this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository); } canUploadFile({ authUser, fieldName, file }: UploadRequest): true { diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index e798de9a1..a17033233 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -14,6 +14,7 @@ import { newAssetRepositoryMock, newJobRepositoryMock, newMediaRepositoryMock, + newMoveRepositoryMock, newPersonRepositoryMock, newStorageRepositoryMock, newSystemConfigRepositoryMock, @@ -25,6 +26,7 @@ import { IAssetRepository, IJobRepository, IMediaRepository, + IMoveRepository, IPersonRepository, IStorageRepository, ISystemConfigRepository, @@ -38,6 +40,7 @@ describe(MediaService.name, () => { let configMock: jest.Mocked; let jobMock: jest.Mocked; let mediaMock: jest.Mocked; + let moveMock: jest.Mocked; let personMock: jest.Mocked; let storageMock: jest.Mocked; @@ -46,10 +49,11 @@ describe(MediaService.name, () => { configMock = newSystemConfigRepositoryMock(); jobMock = newJobRepositoryMock(); mediaMock = newMediaRepositoryMock(); + moveMock = newMoveRepositoryMock(); personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); - sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock); + sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock); }); it('should be defined', () => { diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 9d06c9ad0..09ac53a7f 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -1,4 +1,12 @@ -import { AssetEntity, AssetType, Colorspace, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities'; +import { + AssetEntity, + AssetPathType, + AssetType, + Colorspace, + TranscodeHWAccel, + TranscodePolicy, + VideoCodec, +} from '@app/infra/entities'; import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common'; import { usePagination } from '../domain.util'; import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job'; @@ -7,6 +15,7 @@ import { IAssetRepository, IJobRepository, IMediaRepository, + IMoveRepository, IPersonRepository, IStorageRepository, ISystemConfigRepository, @@ -32,9 +41,10 @@ export class MediaService { @Inject(IMediaRepository) private mediaRepository: IMediaRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, + @Inject(IMoveRepository) moveRepository: IMoveRepository, ) { this.configCore = SystemConfigCore.create(configRepository); - this.storageCore = new StorageCore(this.storageRepository); + this.storageCore = new StorageCore(this.storageRepository, assetRepository, moveRepository, personRepository); } async handleQueueGenerateThumbnails({ force }: IBaseJob) { @@ -108,29 +118,9 @@ export class MediaService { return false; } - if (asset.resizePath) { - const resizePath = this.ensureThumbnailPath(asset, 'jpeg'); - if (asset.resizePath !== resizePath) { - await this.storageRepository.moveFile(asset.resizePath, resizePath); - await this.assetRepository.save({ id: asset.id, resizePath }); - } - } - - if (asset.webpPath) { - const webpPath = this.ensureThumbnailPath(asset, 'webp'); - if (asset.webpPath !== webpPath) { - await this.storageRepository.moveFile(asset.webpPath, webpPath); - await this.assetRepository.save({ id: asset.id, webpPath }); - } - } - - if (asset.encodedVideoPath) { - const encodedVideoPath = this.ensureEncodedVideoPath(asset, 'mp4'); - if (asset.encodedVideoPath !== encodedVideoPath) { - await this.storageRepository.moveFile(asset.encodedVideoPath, encodedVideoPath); - await this.assetRepository.save({ id: asset.id, encodedVideoPath }); - } - } + await this.storageCore.moveAssetFile(asset, AssetPathType.JPEG_THUMBNAIL); + await this.storageCore.moveAssetFile(asset, AssetPathType.WEBP_THUMBNAIL); + await this.storageCore.moveAssetFile(asset, AssetPathType.ENCODED_VIDEO); return true; } @@ -146,15 +136,33 @@ export class MediaService { return true; } - async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { - let path; + private async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { + const { thumbnail, ffmpeg } = await this.configCore.getConfig(); + const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize; + const path = + format === 'jpeg' ? this.storageCore.getLargeThumbnailPath(asset) : this.storageCore.getSmallThumbnailPath(asset); + this.storageCore.ensureFolders(path); + switch (asset.type) { case AssetType.IMAGE: - path = await this.generateImageThumbnail(asset, format); + const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : thumbnail.colorspace; + const thumbnailOptions = { format, size, colorspace, quality: thumbnail.quality }; + await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions); break; + case AssetType.VIDEO: - path = await this.generateVideoThumbnail(asset, format); + const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); + const mainVideoStream = this.getMainStream(videoStreams); + if (!mainVideoStream) { + this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`); + return; + } + const mainAudioStream = this.getMainStream(audioStreams); + const config = { ...ffmpeg, targetResolution: size.toString() }; + const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); + await this.mediaRepository.transcode(asset.originalPath, path, options); break; + default: throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`); } @@ -164,33 +172,6 @@ export class MediaService { return path; } - async generateImageThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { - const { thumbnail } = await this.configCore.getConfig(); - const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize; - const path = this.ensureThumbnailPath(asset, format); - const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : thumbnail.colorspace; - const thumbnailOptions = { format, size, colorspace, quality: thumbnail.quality }; - await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions); - return path; - } - - async generateVideoThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { - const { ffmpeg, thumbnail } = await this.configCore.getConfig(); - const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize; - const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); - const mainVideoStream = this.getMainStream(videoStreams); - if (!mainVideoStream) { - this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`); - return; - } - const mainAudioStream = this.getMainStream(audioStreams); - const path = this.ensureThumbnailPath(asset, format); - const config = { ...ffmpeg, targetResolution: size.toString() }; - const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); - await this.mediaRepository.transcode(asset.originalPath, path, options); - return path; - } - async handleGenerateWebpThumbnail({ id }: IEntityJob) { const [asset] = await this.assetRepository.getByIds([id]); if (!asset) { @@ -239,7 +220,8 @@ export class MediaService { } const input = asset.originalPath; - const output = this.ensureEncodedVideoPath(asset, 'mp4'); + const output = this.storageCore.getEncodedVideoPath(asset); + this.storageCore.ensureFolders(output); const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input); const mainVideoStream = this.getMainStream(videoStreams); @@ -382,14 +364,6 @@ export class MediaService { return handler; } - ensureThumbnailPath(asset: AssetEntity, extension: string): string { - return this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.${extension}`); - } - - ensureEncodedVideoPath(asset: AssetEntity, extension: string): string { - return this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.${extension}`); - } - isSRGB(asset: AssetEntity): boolean { const { colorspace, profileDescription, bitsPerSample } = asset.exifInfo ?? {}; if (colorspace || profileDescription) { diff --git a/server/src/domain/metadata/metadata.service.spec.ts b/server/src/domain/metadata/metadata.service.spec.ts index 4f8b6d017..1096db613 100644 --- a/server/src/domain/metadata/metadata.service.spec.ts +++ b/server/src/domain/metadata/metadata.service.spec.ts @@ -6,6 +6,8 @@ import { newCryptoRepositoryMock, newJobRepositoryMock, newMetadataRepositoryMock, + newMoveRepositoryMock, + newPersonRepositoryMock, newStorageRepositoryMock, newSystemConfigRepositoryMock, } from '@test'; @@ -19,6 +21,8 @@ import { ICryptoRepository, IJobRepository, IMetadataRepository, + IMoveRepository, + IPersonRepository, IStorageRepository, ISystemConfigRepository, ImmichTags, @@ -34,6 +38,8 @@ describe(MetadataService.name, () => { let cryptoRepository: jest.Mocked; let jobMock: jest.Mocked; let metadataMock: jest.Mocked; + let moveMock: jest.Mocked; + let personMock: jest.Mocked; let storageMock: jest.Mocked; let sut: MetadataService; @@ -44,9 +50,21 @@ describe(MetadataService.name, () => { cryptoRepository = newCryptoRepositoryMock(); jobMock = newJobRepositoryMock(); metadataMock = newMetadataRepositoryMock(); + moveMock = newMoveRepositoryMock(); + personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); - sut = new MetadataService(albumMock, assetMock, cryptoRepository, jobMock, metadataMock, storageMock, configMock); + sut = new MetadataService( + albumMock, + assetMock, + cryptoRepository, + jobMock, + metadataMock, + storageMock, + configMock, + moveMock, + personMock, + ); }); it('should be defined', () => { diff --git a/server/src/domain/metadata/metadata.service.ts b/server/src/domain/metadata/metadata.service.ts index 7970b0726..cc03537f5 100644 --- a/server/src/domain/metadata/metadata.service.ts +++ b/server/src/domain/metadata/metadata.service.ts @@ -12,13 +12,15 @@ import { ICryptoRepository, IJobRepository, IMetadataRepository, + IMoveRepository, + IPersonRepository, IStorageRepository, ISystemConfigRepository, ImmichTags, WithProperty, WithoutProperty, } from '../repositories'; -import { StorageCore, StorageFolder } from '../storage'; +import { StorageCore } from '../storage'; import { FeatureFlag, SystemConfigCore } from '../system-config'; interface DirectoryItem { @@ -73,9 +75,11 @@ export class MetadataService { @Inject(IMetadataRepository) private repository: IMetadataRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, + @Inject(IMoveRepository) moveRepository: IMoveRepository, + @Inject(IPersonRepository) personRepository: IPersonRepository, ) { - this.storageCore = new StorageCore(storageRepository); this.configCore = SystemConfigCore.create(configRepository); + this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository); this.configCore.config$.subscribe(() => this.init()); } @@ -296,7 +300,7 @@ export class MetadataService { localDateTime: createdAt, checksum, ownerId: asset.ownerId, - originalPath: this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`), + originalPath: this.storageCore.getAndroidMotionPath(asset), originalFileName: asset.originalFileName, isVisible: false, isReadOnly: false, diff --git a/server/src/domain/person/person.service.spec.ts b/server/src/domain/person/person.service.spec.ts index 36aec20c5..34e305526 100644 --- a/server/src/domain/person/person.service.spec.ts +++ b/server/src/domain/person/person.service.spec.ts @@ -10,6 +10,7 @@ import { newJobRepositoryMock, newMachineLearningRepositoryMock, newMediaRepositoryMock, + newMoveRepositoryMock, newPersonRepositoryMock, newSearchRepositoryMock, newStorageRepositoryMock, @@ -23,6 +24,7 @@ import { IJobRepository, IMachineLearningRepository, IMediaRepository, + IMoveRepository, IPersonRepository, ISearchRepository, IStorageRepository, @@ -91,6 +93,7 @@ describe(PersonService.name, () => { let jobMock: jest.Mocked; let machineLearningMock: jest.Mocked; let mediaMock: jest.Mocked; + let moveMock: jest.Mocked; let personMock: jest.Mocked; let searchMock: jest.Mocked; let storageMock: jest.Mocked; @@ -102,6 +105,7 @@ describe(PersonService.name, () => { configMock = newSystemConfigRepositoryMock(); jobMock = newJobRepositoryMock(); machineLearningMock = newMachineLearningRepositoryMock(); + moveMock = newMoveRepositoryMock(); mediaMock = newMediaRepositoryMock(); personMock = newPersonRepositoryMock(); searchMock = newSearchRepositoryMock(); @@ -110,6 +114,7 @@ describe(PersonService.name, () => { accessMock, assetMock, machineLearningMock, + moveMock, mediaMock, personMock, searchMock, @@ -547,19 +552,19 @@ describe(PersonService.name, () => { it('should generate a thumbnail', async () => { personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.middle.assetId }); personMock.getFacesByIds.mockResolvedValue([faceStub.middle]); - assetMock.getByIds.mockResolvedValue([assetStub.image]); + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); await sut.handleGeneratePersonThumbnail({ id: 'person-1' }); expect(assetMock.getByIds).toHaveBeenCalledWith([faceStub.middle.assetId]); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/pe/rs'); - expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', { + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/admin_id/pe/rs'); + expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/admin-id/thumbs/path.jpg', { left: 95, top: 95, width: 110, height: 110, }); - expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', { + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', { format: 'jpeg', size: 250, quality: 80, @@ -567,7 +572,7 @@ describe(PersonService.name, () => { }); expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', - thumbnailPath: 'upload/thumbs/user-id/pe/rs/person-1.jpeg', + thumbnailPath: 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', }); }); @@ -584,7 +589,7 @@ describe(PersonService.name, () => { width: 510, height: 510, }); - expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', { + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', { format: 'jpeg', size: 250, quality: 80, @@ -595,17 +600,17 @@ describe(PersonService.name, () => { it('should generate a thumbnail without overflowing', async () => { personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.end.assetId }); personMock.getFacesByIds.mockResolvedValue([faceStub.end]); - assetMock.getByIds.mockResolvedValue([assetStub.image]); + assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); await sut.handleGeneratePersonThumbnail({ id: 'person-1' }); - expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', { + expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/admin-id/thumbs/path.jpg', { left: 297, top: 297, width: 202, height: 202, }); - expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', { + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', { format: 'jpeg', size: 250, quality: 80, diff --git a/server/src/domain/person/person.service.ts b/server/src/domain/person/person.service.ts index 528a6dd98..b806862df 100644 --- a/server/src/domain/person/person.service.ts +++ b/server/src/domain/person/person.service.ts @@ -1,4 +1,5 @@ import { PersonEntity } from '@app/infra/entities'; +import { PersonPathType } from '@app/infra/entities/move.entity'; import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; import { AccessCore, Permission } from '../access'; import { AssetResponseDto, BulkIdErrorReason, BulkIdResponseDto, mapAsset } from '../asset'; @@ -15,6 +16,7 @@ import { IJobRepository, IMachineLearningRepository, IMediaRepository, + IMoveRepository, IPersonRepository, ISearchRepository, IStorageRepository, @@ -23,7 +25,7 @@ import { UpdateFacesData, WithoutProperty, } from '../repositories'; -import { StorageCore, StorageFolder } from '../storage'; +import { StorageCore } from '../storage'; import { SystemConfigCore } from '../system-config'; import { MergePersonDto, @@ -46,6 +48,7 @@ export class PersonService { @Inject(IAccessRepository) accessRepository: IAccessRepository, @Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(IMachineLearningRepository) private machineLearningRepository: IMachineLearningRepository, + @Inject(IMoveRepository) moveRepository: IMoveRepository, @Inject(IMediaRepository) private mediaRepository: IMediaRepository, @Inject(IPersonRepository) private repository: IPersonRepository, @Inject(ISearchRepository) private searchRepository: ISearchRepository, @@ -54,8 +57,8 @@ export class PersonService { @Inject(IJobRepository) private jobRepository: IJobRepository, ) { this.access = new AccessCore(accessRepository); - this.storageCore = new StorageCore(storageRepository); this.configCore = SystemConfigCore.create(configRepository); + this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, repository); } async getAll(authUser: AuthUserDto, dto: PersonSearchDto): Promise { @@ -268,11 +271,7 @@ export class PersonService { return false; } - const path = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, person.ownerId, `${id}.jpeg`); - if (person.thumbnailPath && person.thumbnailPath !== path) { - await this.storageRepository.moveFile(person.thumbnailPath, path); - await this.repository.update({ id, thumbnailPath: path }); - } + await this.storageCore.movePersonFile(person, PersonPathType.FACE); return true; } @@ -310,8 +309,8 @@ export class PersonService { } this.logger.verbose(`Cropping face for person: ${personId}`); - - const thumbnailPath = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${personId}.jpeg`); + const thumbnailPath = this.storageCore.getPersonThumbnailPath(person); + this.storageCore.ensureFolders(thumbnailPath); const halfWidth = (x2 - x1) / 2; const halfHeight = (y2 - y1) / 2; diff --git a/server/src/domain/repositories/index.ts b/server/src/domain/repositories/index.ts index 2a6ef6277..28f1cf6a5 100644 --- a/server/src/domain/repositories/index.ts +++ b/server/src/domain/repositories/index.ts @@ -10,6 +10,7 @@ export * from './library.repository'; export * from './machine-learning.repository'; export * from './media.repository'; export * from './metadata.repository'; +export * from './move.repository'; export * from './partner.repository'; export * from './person.repository'; export * from './search.repository'; diff --git a/server/src/domain/repositories/move.repository.ts b/server/src/domain/repositories/move.repository.ts new file mode 100644 index 000000000..20caa117f --- /dev/null +++ b/server/src/domain/repositories/move.repository.ts @@ -0,0 +1,12 @@ +import { MoveEntity, PathType } from '@app/infra/entities'; + +export const IMoveRepository = 'IMoveRepository'; + +export type MoveCreate = Pick & Partial; + +export interface IMoveRepository { + create(entity: MoveCreate): Promise; + getByEntity(entityId: string, pathType: PathType): Promise; + update(entity: Partial): Promise; + delete(move: MoveEntity): Promise; +} diff --git a/server/src/domain/server-info/server-info.service.spec.ts b/server/src/domain/server-info/server-info.service.spec.ts index 53115594c..8655f7cc5 100644 --- a/server/src/domain/server-info/server-info.service.spec.ts +++ b/server/src/domain/server-info/server-info.service.spec.ts @@ -1,20 +1,40 @@ -import { newStorageRepositoryMock, newSystemConfigRepositoryMock, newUserRepositoryMock } from '@test'; +import { + newAssetRepositoryMock, + newMoveRepositoryMock, + newPersonRepositoryMock, + newStorageRepositoryMock, + newSystemConfigRepositoryMock, + newUserRepositoryMock, +} from '@test'; import { serverVersion } from '../domain.constant'; -import { IStorageRepository, ISystemConfigRepository, IUserRepository } from '../repositories'; +import { + IAssetRepository, + IMoveRepository, + IPersonRepository, + IStorageRepository, + ISystemConfigRepository, + IUserRepository, +} from '../repositories'; import { ServerInfoService } from './server-info.service'; describe(ServerInfoService.name, () => { let sut: ServerInfoService; + let assetMock: jest.Mocked; let configMock: jest.Mocked; + let moveMock: jest.Mocked; + let personMock: jest.Mocked; let storageMock: jest.Mocked; let userMock: jest.Mocked; beforeEach(() => { + assetMock = newAssetRepositoryMock(); configMock = newSystemConfigRepositoryMock(); + moveMock = newMoveRepositoryMock(); + personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); userMock = newUserRepositoryMock(); - sut = new ServerInfoService(configMock, userMock, storageMock); + sut = new ServerInfoService(assetMock, configMock, moveMock, personMock, userMock, storageMock); }); it('should work', () => { diff --git a/server/src/domain/server-info/server-info.service.ts b/server/src/domain/server-info/server-info.service.ts index 0d17ca391..69a925e86 100644 --- a/server/src/domain/server-info/server-info.service.ts +++ b/server/src/domain/server-info/server-info.service.ts @@ -1,7 +1,15 @@ import { Inject, Injectable } from '@nestjs/common'; import { mimeTypes, serverVersion } from '../domain.constant'; import { asHumanReadable } from '../domain.util'; -import { IStorageRepository, ISystemConfigRepository, IUserRepository, UserStatsQueryResponse } from '../repositories'; +import { + IAssetRepository, + IMoveRepository, + IPersonRepository, + IStorageRepository, + ISystemConfigRepository, + IUserRepository, + UserStatsQueryResponse, +} from '../repositories'; import { StorageCore, StorageFolder } from '../storage'; import { SystemConfigCore } from '../system-config'; import { @@ -20,12 +28,15 @@ export class ServerInfoService { private storageCore: StorageCore; constructor( + @Inject(IAssetRepository) assetRepository: IAssetRepository, @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, + @Inject(IMoveRepository) moveRepository: IMoveRepository, + @Inject(IPersonRepository) personRepository: IPersonRepository, @Inject(IUserRepository) private userRepository: IUserRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, ) { this.configCore = SystemConfigCore.create(configRepository); - this.storageCore = new StorageCore(storageRepository); + this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository); } async getInfo(): Promise { diff --git a/server/src/domain/storage-template/storage-template.service.spec.ts b/server/src/domain/storage-template/storage-template.service.spec.ts index f5a5a8b2a..8f6833699 100644 --- a/server/src/domain/storage-template/storage-template.service.spec.ts +++ b/server/src/domain/storage-template/storage-template.service.spec.ts @@ -1,13 +1,23 @@ +import { AssetPathType } from '@app/infra/entities'; import { assetStub, newAssetRepositoryMock, + newMoveRepositoryMock, + newPersonRepositoryMock, newStorageRepositoryMock, newSystemConfigRepositoryMock, newUserRepositoryMock, userStub, } from '@test'; import { when } from 'jest-when'; -import { IAssetRepository, IStorageRepository, ISystemConfigRepository, IUserRepository } from '../repositories'; +import { + IAssetRepository, + IMoveRepository, + IPersonRepository, + IStorageRepository, + ISystemConfigRepository, + IUserRepository, +} from '../repositories'; import { defaults } from '../system-config/system-config.core'; import { StorageTemplateService } from './storage-template.service'; @@ -15,6 +25,8 @@ describe(StorageTemplateService.name, () => { let sut: StorageTemplateService; let assetMock: jest.Mocked; let configMock: jest.Mocked; + let moveMock: jest.Mocked; + let personMock: jest.Mocked; let storageMock: jest.Mocked; let userMock: jest.Mocked; @@ -25,10 +37,12 @@ describe(StorageTemplateService.name, () => { beforeEach(async () => { assetMock = newAssetRepositoryMock(); configMock = newSystemConfigRepositoryMock(); + moveMock = newMoveRepositoryMock(); + personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); userMock = newUserRepositoryMock(); - sut = new StorageTemplateService(assetMock, configMock, defaults, storageMock, userMock); + sut = new StorageTemplateService(assetMock, configMock, defaults, moveMock, personMock, storageMock, userMock); }); describe('handleMigrationSingle', () => { @@ -86,6 +100,13 @@ describe(StorageTemplateService.name, () => { }); assetMock.save.mockResolvedValue(assetStub.image); userMock.getList.mockResolvedValue([userStub.user1]); + moveMock.create.mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg', + }); when(storageMock.checkFileExists) .calledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg') @@ -153,6 +174,13 @@ describe(StorageTemplateService.name, () => { }); assetMock.save.mockResolvedValue(assetStub.image); userMock.getList.mockResolvedValue([userStub.user1]); + moveMock.create.mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + }); await sut.handleMigration(); @@ -174,6 +202,13 @@ describe(StorageTemplateService.name, () => { }); assetMock.save.mockResolvedValue(assetStub.image); userMock.getList.mockResolvedValue([userStub.storageLabel]); + moveMock.create.mockResolvedValue({ + id: '123', + entityId: assetStub.image.id, + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', + }); await sut.handleMigration(); @@ -194,6 +229,13 @@ describe(StorageTemplateService.name, () => { hasNextPage: false, }); storageMock.moveFile.mockRejectedValue(new Error('Read only system')); + moveMock.create.mockResolvedValue({ + id: 'move-123', + entityId: '123', + pathType: AssetPathType.ORIGINAL, + oldPath: assetStub.image.originalPath, + newPath: '', + }); userMock.getList.mockResolvedValue([userStub.user1]); await sut.handleMigration(); @@ -206,27 +248,6 @@ describe(StorageTemplateService.name, () => { expect(assetMock.save).not.toHaveBeenCalled(); }); - it('should move the asset back if the database fails', async () => { - assetMock.getAll.mockResolvedValue({ - items: [assetStub.image], - hasNextPage: false, - }); - assetMock.save.mockRejectedValue('Connection Error!'); - userMock.getList.mockResolvedValue([userStub.user1]); - - await sut.handleMigration(); - - expect(assetMock.getAll).toHaveBeenCalled(); - expect(assetMock.save).toHaveBeenCalledWith({ - id: assetStub.image.id, - originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg', - }); - expect(storageMock.moveFile.mock.calls).toEqual([ - ['/original/path.jpg', 'upload/library/user-id/2023/2023-02-23/asset-id.jpg'], - ['upload/library/user-id/2023/2023-02-23/asset-id.jpg', '/original/path.jpg'], - ]); - }); - it('should not move read-only asset', async () => { assetMock.getAll.mockResolvedValue({ items: [ diff --git a/server/src/domain/storage-template/storage-template.service.ts b/server/src/domain/storage-template/storage-template.service.ts index 60439c575..6681e6062 100644 --- a/server/src/domain/storage-template/storage-template.service.ts +++ b/server/src/domain/storage-template/storage-template.service.ts @@ -1,4 +1,4 @@ -import { AssetEntity, AssetType, SystemConfig } from '@app/infra/entities'; +import { AssetEntity, AssetPathType, AssetType, SystemConfig } from '@app/infra/entities'; import { Inject, Injectable, Logger } from '@nestjs/common'; import handlebar from 'handlebars'; import * as luxon from 'luxon'; @@ -6,7 +6,14 @@ import path from 'node:path'; import sanitize from 'sanitize-filename'; import { getLivePhotoMotionFilename, usePagination } from '../domain.util'; import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job'; -import { IAssetRepository, IStorageRepository, ISystemConfigRepository, IUserRepository } from '../repositories'; +import { + IAssetRepository, + IMoveRepository, + IPersonRepository, + IStorageRepository, + ISystemConfigRepository, + IUserRepository, +} from '../repositories'; import { StorageCore, StorageFolder } from '../storage'; import { INITIAL_SYSTEM_CONFIG, @@ -36,6 +43,8 @@ export class StorageTemplateService { @Inject(IAssetRepository) private assetRepository: IAssetRepository, @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, @Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig, + @Inject(IMoveRepository) moveRepository: IMoveRepository, + @Inject(IPersonRepository) personRepository: IPersonRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(IUserRepository) private userRepository: IUserRepository, ) { @@ -43,7 +52,7 @@ export class StorageTemplateService { this.configCore = SystemConfigCore.create(configRepository); this.configCore.addValidator((config) => this.validate(config)); this.configCore.config$.subscribe((config) => this.onConfig(config)); - this.storageCore = new StorageCore(storageRepository); + this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository); } async handleMigrationSingle({ id }: IEntityJob) { @@ -90,51 +99,29 @@ export class StorageTemplateService { } async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) { - if (asset.isReadOnly || asset.isExternal) { + if (asset.isReadOnly || asset.isExternal || this.storageCore.isAndroidMotionPath(asset.originalPath)) { // External assets are not affected by storage template // TODO: shouldn't this only apply to external assets? return; } - const destination = await this.getTemplatePath(asset, metadata); - if (asset.originalPath !== destination) { - const source = asset.originalPath; + const { id, sidecarPath, originalPath } = asset; + const oldPath = originalPath; + const newPath = await this.getTemplatePath(asset, metadata); - let sidecarMoved = false; - try { - await this.storageRepository.moveFile(asset.originalPath, destination); - - let sidecarDestination; - try { - if (asset.sidecarPath) { - sidecarDestination = `${destination}.xmp`; - await this.storageRepository.moveFile(asset.sidecarPath, sidecarDestination); - sidecarMoved = true; - } - - await this.assetRepository.save({ id: asset.id, originalPath: destination, sidecarPath: sidecarDestination }); - asset.originalPath = destination; - asset.sidecarPath = sidecarDestination || null; - } catch (error: any) { - this.logger.warn( - `Unable to save new originalPath to database, undoing move for path ${asset.originalPath} - filename ${asset.originalFileName} - id ${asset.id}`, - error?.stack, - ); - - // Either sidecar move failed or the save failed. Either way, move media back - await this.storageRepository.moveFile(destination, source); - - if (asset.sidecarPath && sidecarDestination && sidecarMoved) { - // If the sidecar was moved, that means the saved failed. So move both the sidecar and the - // media back into their original positions - await this.storageRepository.moveFile(sidecarDestination, asset.sidecarPath); - } - } - } catch (error: any) { - this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination }); + try { + await this.storageCore.moveFile({ entityId: id, pathType: AssetPathType.ORIGINAL, oldPath, newPath }); + if (sidecarPath) { + await this.storageCore.moveFile({ + entityId: id, + pathType: AssetPathType.SIDECAR, + oldPath: sidecarPath, + newPath: `${newPath}.xmp`, + }); } + } catch (error: any) { + this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, oldPath, newPath }); } - return asset; } private async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise { diff --git a/server/src/domain/storage/storage.core.ts b/server/src/domain/storage/storage.core.ts index 15bdd0405..249b2857f 100644 --- a/server/src/domain/storage/storage.core.ts +++ b/server/src/domain/storage/storage.core.ts @@ -1,6 +1,8 @@ -import { join } from 'node:path'; +import { AssetEntity, AssetPathType, PathType, PersonEntity, PersonPathType } from '@app/infra/entities'; +import { Logger } from '@nestjs/common'; +import { dirname, join } from 'node:path'; import { APP_MEDIA_LOCATION } from '../domain.constant'; -import { IStorageRepository } from '../repositories'; +import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories'; export enum StorageFolder { ENCODED_VIDEO = 'encoded-video', @@ -10,13 +12,26 @@ export enum StorageFolder { THUMBNAILS = 'thumbs', } -export class StorageCore { - constructor(private repository: IStorageRepository) {} +export interface MoveRequest { + entityId: string; + pathType: PathType; + oldPath: string | null; + newPath: string; +} - getFolderLocation( - folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS, - userId: string, - ) { +type GeneratedAssetPath = AssetPathType.JPEG_THUMBNAIL | AssetPathType.WEBP_THUMBNAIL | AssetPathType.ENCODED_VIDEO; + +export class StorageCore { + private logger = new Logger(StorageCore.name); + + constructor( + private repository: IStorageRepository, + private assetRepository: IAssetRepository, + private moveRepository: IMoveRepository, + private personRepository: IPersonRepository, + ) {} + + getFolderLocation(folder: StorageFolder, userId: string) { return join(this.getBaseFolder(folder), userId); } @@ -28,21 +43,119 @@ export class StorageCore { return join(APP_MEDIA_LOCATION, folder); } - ensurePath( - folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS, - ownerId: string, - fileName: string, - ): string { - const folderPath = join( - this.getFolderLocation(folder, ownerId), - fileName.substring(0, 2), - fileName.substring(2, 4), - ); - this.repository.mkdirSync(folderPath); - return join(folderPath, fileName); + getPersonThumbnailPath(person: PersonEntity) { + return this.getNestedPath(StorageFolder.THUMBNAILS, person.ownerId, `${person.id}.jpeg`); + } + + getLargeThumbnailPath(asset: AssetEntity) { + return this.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.jpeg`); + } + + getSmallThumbnailPath(asset: AssetEntity) { + return this.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.webp`); + } + + getEncodedVideoPath(asset: AssetEntity) { + return this.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.mp4`); + } + + getAndroidMotionPath(asset: AssetEntity) { + return this.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`); + } + + isAndroidMotionPath(originalPath: string) { + return originalPath.startsWith(this.getBaseFolder(StorageFolder.ENCODED_VIDEO)); + } + + async moveAssetFile(asset: AssetEntity, pathType: GeneratedAssetPath) { + const { id: entityId, resizePath, webpPath, encodedVideoPath } = asset; + switch (pathType) { + case AssetPathType.JPEG_THUMBNAIL: + return this.moveFile({ entityId, pathType, oldPath: resizePath, newPath: this.getLargeThumbnailPath(asset) }); + case AssetPathType.WEBP_THUMBNAIL: + return this.moveFile({ entityId, pathType, oldPath: webpPath, newPath: this.getSmallThumbnailPath(asset) }); + case AssetPathType.ENCODED_VIDEO: + return this.moveFile({ + entityId, + pathType, + oldPath: encodedVideoPath, + newPath: this.getEncodedVideoPath(asset), + }); + } + } + + async movePersonFile(person: PersonEntity, pathType: PersonPathType) { + const { id: entityId, thumbnailPath } = person; + switch (pathType) { + case PersonPathType.FACE: + await this.moveFile({ + entityId, + pathType, + oldPath: thumbnailPath, + newPath: this.getPersonThumbnailPath(person), + }); + } + } + + async moveFile(request: MoveRequest) { + const { entityId, pathType, oldPath, newPath } = request; + if (!oldPath || oldPath === newPath) { + return; + } + + this.ensureFolders(newPath); + + let move = await this.moveRepository.getByEntity(entityId, pathType); + if (move) { + this.logger.log(`Attempting to finish incomplete move: ${move.oldPath} => ${move.newPath}`); + const oldPathExists = await this.repository.checkFileExists(move.oldPath); + const newPathExists = await this.repository.checkFileExists(move.newPath); + const actualPath = newPathExists ? move.newPath : oldPathExists ? move.oldPath : null; + if (!actualPath) { + this.logger.warn('Unable to complete move. File does not exist at either location.'); + return; + } + + this.logger.log(`Found file at ${actualPath === move.oldPath ? 'old' : 'new'} location`); + + move = await this.moveRepository.update({ id: move.id, oldPath: actualPath, newPath }); + } else { + move = await this.moveRepository.create({ entityId, pathType, oldPath, newPath }); + } + + if (move.oldPath !== newPath) { + await this.repository.moveFile(move.oldPath, newPath); + } + await this.savePath(pathType, entityId, newPath); + await this.moveRepository.delete(move); + } + + ensureFolders(input: string) { + this.repository.mkdirSync(dirname(input)); } removeEmptyDirs(folder: StorageFolder) { return this.repository.removeEmptyDirs(this.getBaseFolder(folder)); } + + private savePath(pathType: PathType, id: string, newPath: string) { + switch (pathType) { + case AssetPathType.ORIGINAL: + return this.assetRepository.save({ id, originalPath: newPath }); + case AssetPathType.JPEG_THUMBNAIL: + return this.assetRepository.save({ id, resizePath: newPath }); + case AssetPathType.WEBP_THUMBNAIL: + return this.assetRepository.save({ id, webpPath: newPath }); + case AssetPathType.ENCODED_VIDEO: + return this.assetRepository.save({ id, encodedVideoPath: newPath }); + case AssetPathType.SIDECAR: + return this.assetRepository.save({ id, sidecarPath: newPath }); + case PersonPathType.FACE: + return this.personRepository.update({ id, thumbnailPath: newPath }); + } + } + + private getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string { + return join(this.getFolderLocation(folder, ownerId), filename.substring(0, 2), filename.substring(2, 4), filename); + } } diff --git a/server/src/domain/storage/storage.service.spec.ts b/server/src/domain/storage/storage.service.spec.ts index 0c5531e5f..e197dee4a 100644 --- a/server/src/domain/storage/storage.service.spec.ts +++ b/server/src/domain/storage/storage.service.spec.ts @@ -1,14 +1,25 @@ -import { newStorageRepositoryMock } from '@test'; -import { IStorageRepository } from '../repositories'; +import { + newAssetRepositoryMock, + newMoveRepositoryMock, + newPersonRepositoryMock, + newStorageRepositoryMock, +} from '@test'; +import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories'; import { StorageService } from './storage.service'; describe(StorageService.name, () => { let sut: StorageService; + let assetMock: jest.Mocked; + let moveMock: jest.Mocked; + let personMock: jest.Mocked; let storageMock: jest.Mocked; beforeEach(async () => { + assetMock = newAssetRepositoryMock(); + moveMock = newMoveRepositoryMock(); + personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); - sut = new StorageService(storageMock); + sut = new StorageService(assetMock, moveMock, personMock, storageMock); }); it('should work', () => { diff --git a/server/src/domain/storage/storage.service.ts b/server/src/domain/storage/storage.service.ts index 001c0c869..629811313 100644 --- a/server/src/domain/storage/storage.service.ts +++ b/server/src/domain/storage/storage.service.ts @@ -1,6 +1,6 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; import { IDeleteFilesJob } from '../job'; -import { IStorageRepository } from '../repositories'; +import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories'; import { StorageCore, StorageFolder } from './storage.core'; @Injectable() @@ -8,8 +8,13 @@ export class StorageService { private logger = new Logger(StorageService.name); private storageCore: StorageCore; - constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) { - this.storageCore = new StorageCore(storageRepository); + constructor( + @Inject(IAssetRepository) assetRepository: IAssetRepository, + @Inject(IMoveRepository) private moveRepository: IMoveRepository, + @Inject(IPersonRepository) personRepository: IPersonRepository, + @Inject(IStorageRepository) private storageRepository: IStorageRepository, + ) { + this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository); } init() { diff --git a/server/src/domain/user/user.service.spec.ts b/server/src/domain/user/user.service.spec.ts index d93bc6677..f54ee603d 100644 --- a/server/src/domain/user/user.service.spec.ts +++ b/server/src/domain/user/user.service.spec.ts @@ -11,6 +11,8 @@ import { newCryptoRepositoryMock, newJobRepositoryMock, newLibraryRepositoryMock, + newMoveRepositoryMock, + newPersonRepositoryMock, newStorageRepositoryMock, newUserRepositoryMock, userStub, @@ -24,6 +26,8 @@ import { ICryptoRepository, IJobRepository, ILibraryRepository, + IMoveRepository, + IPersonRepository, IStorageRepository, IUserRepository, } from '../repositories'; @@ -135,18 +139,32 @@ describe(UserService.name, () => { let assetMock: jest.Mocked; let jobMock: jest.Mocked; let libraryMock: jest.Mocked; + let moveMock: jest.Mocked; + let personMock: jest.Mocked; let storageMock: jest.Mocked; beforeEach(async () => { - cryptoRepositoryMock = newCryptoRepositoryMock(); albumMock = newAlbumRepositoryMock(); assetMock = newAssetRepositoryMock(); + cryptoRepositoryMock = newCryptoRepositoryMock(); jobMock = newJobRepositoryMock(); libraryMock = newLibraryRepositoryMock(); + moveMock = newMoveRepositoryMock(); + personMock = newPersonRepositoryMock(); storageMock = newStorageRepositoryMock(); userMock = newUserRepositoryMock(); - sut = new UserService(userMock, cryptoRepositoryMock, libraryMock, albumMock, assetMock, jobMock, storageMock); + sut = new UserService( + albumMock, + assetMock, + cryptoRepositoryMock, + jobMock, + libraryMock, + moveMock, + personMock, + storageMock, + userMock, + ); when(userMock.get).calledWith(adminUser.id).mockResolvedValue(adminUser); when(userMock.get).calledWith(adminUser.id, undefined).mockResolvedValue(adminUser); diff --git a/server/src/domain/user/user.service.ts b/server/src/domain/user/user.service.ts index 80be378e4..8664bca50 100644 --- a/server/src/domain/user/user.service.ts +++ b/server/src/domain/user/user.service.ts @@ -10,6 +10,8 @@ import { ICryptoRepository, IJobRepository, ILibraryRepository, + IMoveRepository, + IPersonRepository, IStorageRepository, IUserRepository, } from '../repositories'; @@ -32,15 +34,17 @@ export class UserService { private userCore: UserCore; constructor( - @Inject(IUserRepository) private userRepository: IUserRepository, - @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository, - @Inject(ILibraryRepository) libraryRepository: ILibraryRepository, @Inject(IAlbumRepository) private albumRepository: IAlbumRepository, @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, + @Inject(ILibraryRepository) libraryRepository: ILibraryRepository, + @Inject(IMoveRepository) moveRepository: IMoveRepository, + @Inject(IPersonRepository) personRepository: IPersonRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, + @Inject(IUserRepository) private userRepository: IUserRepository, ) { - this.storageCore = new StorageCore(storageRepository); + this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository); this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository); } diff --git a/server/src/infra/entities/index.ts b/server/src/infra/entities/index.ts index 72d41f287..ef4d635b0 100644 --- a/server/src/infra/entities/index.ts +++ b/server/src/infra/entities/index.ts @@ -5,6 +5,7 @@ import { AssetEntity } from './asset.entity'; import { AuditEntity } from './audit.entity'; import { ExifEntity } from './exif.entity'; import { LibraryEntity } from './library.entity'; +import { MoveEntity } from './move.entity'; import { PartnerEntity } from './partner.entity'; import { PersonEntity } from './person.entity'; import { SharedLinkEntity } from './shared-link.entity'; @@ -21,6 +22,7 @@ export * from './asset.entity'; export * from './audit.entity'; export * from './exif.entity'; export * from './library.entity'; +export * from './move.entity'; export * from './partner.entity'; export * from './person.entity'; export * from './shared-link.entity'; @@ -37,6 +39,7 @@ export const databaseEntities = [ AssetFaceEntity, AuditEntity, ExifEntity, + MoveEntity, PartnerEntity, PersonEntity, SharedLinkEntity, diff --git a/server/src/infra/entities/move.entity.ts b/server/src/infra/entities/move.entity.ts new file mode 100644 index 000000000..daeb7f4b4 --- /dev/null +++ b/server/src/infra/entities/move.entity.ts @@ -0,0 +1,37 @@ +import { Column, Entity, PrimaryGeneratedColumn, Unique } from 'typeorm'; + +@Entity('move_history') +// path lock (per entity) +@Unique('UQ_entityId_pathType', ['entityId', 'pathType']) +// new path lock (global) +@Unique('UQ_newPath', ['newPath']) +export class MoveEntity { + @PrimaryGeneratedColumn('uuid') + id!: string; + + @Column({ type: 'varchar' }) + entityId!: string; + + @Column({ type: 'varchar' }) + pathType!: PathType; + + @Column({ type: 'varchar' }) + oldPath!: string; + + @Column({ type: 'varchar' }) + newPath!: string; +} + +export enum AssetPathType { + ORIGINAL = 'original', + JPEG_THUMBNAIL = 'jpeg_thumbnail', + WEBP_THUMBNAIL = 'webp_thumbnail', + ENCODED_VIDEO = 'encoded_video', + SIDECAR = 'sidecar', +} + +export enum PersonPathType { + FACE = 'face', +} + +export type PathType = AssetPathType | PersonPathType; diff --git a/server/src/infra/infra.module.ts b/server/src/infra/infra.module.ts index 56d70cfb4..ac9cb8484 100644 --- a/server/src/infra/infra.module.ts +++ b/server/src/infra/infra.module.ts @@ -11,6 +11,7 @@ import { IMachineLearningRepository, IMediaRepository, IMetadataRepository, + IMoveRepository, IPartnerRepository, IPersonRepository, ISearchRepository, @@ -44,6 +45,7 @@ import { MachineLearningRepository, MediaRepository, MetadataRepository, + MoveRepository, PartnerRepository, PersonRepository, SharedLinkRepository, @@ -67,6 +69,7 @@ const providers: Provider[] = [ { provide: IKeyRepository, useClass: APIKeyRepository }, { provide: IMachineLearningRepository, useClass: MachineLearningRepository }, { provide: IMetadataRepository, useClass: MetadataRepository }, + { provide: IMoveRepository, useClass: MoveRepository }, { provide: IPartnerRepository, useClass: PartnerRepository }, { provide: IPersonRepository, useClass: PersonRepository }, { provide: ISearchRepository, useClass: TypesenseRepository }, diff --git a/server/src/infra/migrations/1696968880063-AddMoveTable.ts b/server/src/infra/migrations/1696968880063-AddMoveTable.ts new file mode 100644 index 000000000..7ba140d05 --- /dev/null +++ b/server/src/infra/migrations/1696968880063-AddMoveTable.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddMoveTable1696968880063 implements MigrationInterface { + name = 'AddMoveTable1696968880063' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "move_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "entityId" character varying NOT NULL, "pathType" character varying NOT NULL, "oldPath" character varying NOT NULL, "newPath" character varying NOT NULL, CONSTRAINT "UQ_newPath" UNIQUE ("newPath"), CONSTRAINT "UQ_entityId_pathType" UNIQUE ("entityId", "pathType"), CONSTRAINT "PK_af608f132233acf123f2949678d" PRIMARY KEY ("id"))`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "move_history"`); + } + +} diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index 25ad0288d..9c85626c0 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -6,6 +6,7 @@ import { IStorageRepository, mimeTypes, } from '@app/domain'; +import { Logger } from '@nestjs/common'; import archiver from 'archiver'; import { constants, createReadStream, existsSync, mkdirSync } from 'fs'; import fs, { readdir, writeFile } from 'fs/promises'; @@ -17,6 +18,8 @@ import path from 'path'; const moveFile = promisify(mv); export class FilesystemProvider implements IStorageRepository { + private logger = new Logger(FilesystemProvider.name); + createZipStream(): ImmichZipStream { const archive = archiver('zip', { store: true }); @@ -52,6 +55,8 @@ export class FilesystemProvider implements IStorageRepository { writeFile = writeFile; async moveFile(source: string, destination: string): Promise { + this.logger.verbose(`Moving ${source} to ${destination}`); + if (await this.checkFileExists(destination)) { throw new Error(`Destination file already exists: ${destination}`); } diff --git a/server/src/infra/repositories/index.ts b/server/src/infra/repositories/index.ts index bc2d1da2f..5229e610e 100644 --- a/server/src/infra/repositories/index.ts +++ b/server/src/infra/repositories/index.ts @@ -11,6 +11,7 @@ export * from './library.repository'; export * from './machine-learning.repository'; export * from './media.repository'; export * from './metadata.repository'; +export * from './move.repository'; export * from './partner.repository'; export * from './person.repository'; export * from './shared-link.repository'; diff --git a/server/src/infra/repositories/job.repository.ts b/server/src/infra/repositories/job.repository.ts index bd04ccb5f..b4a5ea2a6 100644 --- a/server/src/infra/repositories/job.repository.ts +++ b/server/src/infra/repositories/job.repository.ts @@ -70,6 +70,8 @@ export class JobRepository implements IJobRepository { private getJobOptions(item: JobItem): JobsOptions | null { switch (item.name) { + case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: + return { jobId: item.data.id }; case JobName.GENERATE_PERSON_THUMBNAIL: return { priority: 1 }; diff --git a/server/src/infra/repositories/move.repository.ts b/server/src/infra/repositories/move.repository.ts new file mode 100644 index 000000000..f909b0f20 --- /dev/null +++ b/server/src/infra/repositories/move.repository.ts @@ -0,0 +1,26 @@ +import { IMoveRepository, MoveCreate } from '@app/domain'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { MoveEntity, PathType } from '../entities'; + +@Injectable() +export class MoveRepository implements IMoveRepository { + constructor(@InjectRepository(MoveEntity) private repository: Repository) {} + + create(entity: MoveCreate): Promise { + return this.repository.save(entity); + } + + getByEntity(entityId: string, pathType: PathType): Promise { + return this.repository.findOne({ where: { entityId, pathType } }); + } + + update(entity: Partial): Promise { + return this.repository.save(entity); + } + + delete(move: MoveEntity): Promise { + return this.repository.remove(move); + } +} diff --git a/server/test/fixtures/asset.stub.ts b/server/test/fixtures/asset.stub.ts index 77d0b6bb0..5fef9f6d1 100644 --- a/server/test/fixtures/asset.stub.ts +++ b/server/test/fixtures/asset.stub.ts @@ -116,6 +116,45 @@ export const assetStub = { sidecarPath: null, deletedAt: null, }), + primaryImage: Object.freeze({ + id: 'asset-id', + deviceAssetId: 'device-asset-id', + fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'), + fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'), + owner: userStub.admin, + ownerId: 'admin-id', + deviceId: 'device-id', + originalPath: '/original/path.jpg', + resizePath: '/uploads/admin-id/thumbs/path.jpg', + checksum: Buffer.from('file hash', 'utf8'), + type: AssetType.IMAGE, + webpPath: '/uploads/admin-id/webp/path.ext', + thumbhash: Buffer.from('blablabla', 'base64'), + encodedVideoPath: null, + createdAt: new Date('2023-02-23T05:06:29.716Z'), + updatedAt: new Date('2023-02-23T05:06:29.716Z'), + localDateTime: new Date('2023-02-23T05:06:29.716Z'), + isFavorite: true, + isArchived: false, + isReadOnly: false, + duration: null, + isVisible: true, + isExternal: false, + livePhotoVideo: null, + livePhotoVideoId: null, + isOffline: false, + libraryId: 'library-id', + library: libraryStub.uploadLibrary1, + tags: [], + sharedLinks: [], + originalFileName: 'asset-id.jpg', + faces: [], + deletedAt: null, + sidecarPath: null, + exifInfo: { + fileSizeInByte: 5_000, + } as ExifEntity, + }), image: Object.freeze({ id: 'asset-id', deviceAssetId: 'device-asset-id', diff --git a/server/test/repositories/index.ts b/server/test/repositories/index.ts index 0625023e4..eb91bfa14 100644 --- a/server/test/repositories/index.ts +++ b/server/test/repositories/index.ts @@ -10,6 +10,7 @@ export * from './library.repository.mock'; export * from './machine-learning.repository.mock'; export * from './media.repository.mock'; export * from './metadata.repository.mock'; +export * from './move.repository.mock'; export * from './partner.repository.mock'; export * from './person.repository.mock'; export * from './search.repository.mock'; diff --git a/server/test/repositories/move.repository.mock.ts b/server/test/repositories/move.repository.mock.ts new file mode 100644 index 000000000..e14b0640b --- /dev/null +++ b/server/test/repositories/move.repository.mock.ts @@ -0,0 +1,10 @@ +import { IMoveRepository } from '@app/domain'; + +export const newMoveRepositoryMock = (): jest.Mocked => { + return { + create: jest.fn(), + getByEntity: jest.fn(), + update: jest.fn(), + delete: jest.fn(), + }; +};