beenull/immich
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat(server): harden move file (#4361)

Browse source 
Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Daniel Dietzler 2023-10-11 04:14:44 +02:00 committed by GitHub
parent 332a8d80f2
commit 09bf1c9175
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 564 additions and 190 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
docs/docs/guides/database-queries.md
View file

@ -66,6 +66,10 @@ ORDER BY
"users"."email";
```
```sql title="Failed file movements"
SELECT * FROM "move_history";
```
## Users
```sql title="List"

20
server/src/domain/asset/asset.service.spec.ts
View file

@ -10,6 +10,8 @@ import {
newCommunicationRepositoryMock,
newCryptoRepositoryMock,
newJobRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
} from '@test';
@ -22,6 +24,8 @@ import {
ICommunicationRepository,
ICryptoRepository,
IJobRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
JobItem,
@ -160,6 +164,8 @@ describe(AssetService.name, () => {
let assetMock: jest.Mocked<IAssetRepository>;
let cryptoMock: jest.Mocked<ICryptoRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let communicationMock: jest.Mocked<ICommunicationRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
@ -174,9 +180,21 @@ describe(AssetService.name, () => {
communicationMock = newCommunicationRepositoryMock();
cryptoMock = newCryptoRepositoryMock();
jobMock = newJobRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
configMock = newSystemConfigRepositoryMock();
sut = new AssetService(accessMock, assetMock, cryptoMock, jobMock, configMock, storageMock, communicationMock);
sut = new AssetService(
accessMock,
assetMock,
cryptoMock,
jobMock,
configMock,
moveMock,
personMock,
storageMock,
communicationMock,
);
when(assetMock.getById)
.calledWith(assetStub.livePhotoStillAsset.id)

6
server/src/domain/asset/asset.service.ts
View file

@ -16,6 +16,8 @@ import {
ICommunicationRepository,
ICryptoRepository,
IJobRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
ImmichReadStream,
@ -80,12 +82,14 @@ export class AssetService {
@Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
) {
this.access = new AccessCore(accessRepository);
this.storageCore = new StorageCore(storageRepository);
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
}
canUploadFile({ authUser, fieldName, file }: UploadRequest): true {

6
server/src/domain/media/media.service.spec.ts
View file

@ -14,6 +14,7 @@ import {
newAssetRepositoryMock,
newJobRepositoryMock,
newMediaRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
@ -25,6 +26,7 @@ import {
IAssetRepository,
IJobRepository,
IMediaRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
@ -38,6 +40,7 @@ describe(MediaService.name, () => {
let configMock: jest.Mocked<ISystemConfigRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let mediaMock: jest.Mocked<IMediaRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
@ -46,10 +49,11 @@ describe(MediaService.name, () => {
configMock = newSystemConfigRepositoryMock();
jobMock = newJobRepositoryMock();
mediaMock = newMediaRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock);
sut = new MediaService(assetMock, personMock, jobMock, mediaMock, storageMock, configMock, moveMock);
});
it('should be defined', () => {

104
server/src/domain/media/media.service.ts
View file

@ -1,4 +1,12 @@
import { AssetEntity, AssetType, Colorspace, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
import {
AssetEntity,
AssetPathType,
AssetType,
Colorspace,
TranscodeHWAccel,
TranscodePolicy,
VideoCodec,
} from '@app/infra/entities';
import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
@ -7,6 +15,7 @@ import {
IAssetRepository,
IJobRepository,
IMediaRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
@ -32,9 +41,10 @@ export class MediaService {
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
) {
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(this.storageRepository);
this.storageCore = new StorageCore(this.storageRepository, assetRepository, moveRepository, personRepository);
}
async handleQueueGenerateThumbnails({ force }: IBaseJob) {
@ -108,29 +118,9 @@ export class MediaService {
return false;
}
if (asset.resizePath) {
const resizePath = this.ensureThumbnailPath(asset, 'jpeg');
if (asset.resizePath !== resizePath) {
await this.storageRepository.moveFile(asset.resizePath, resizePath);
await this.assetRepository.save({ id: asset.id, resizePath });
}
}
if (asset.webpPath) {
const webpPath = this.ensureThumbnailPath(asset, 'webp');
if (asset.webpPath !== webpPath) {
await this.storageRepository.moveFile(asset.webpPath, webpPath);
await this.assetRepository.save({ id: asset.id, webpPath });
}
}
if (asset.encodedVideoPath) {
const encodedVideoPath = this.ensureEncodedVideoPath(asset, 'mp4');
if (asset.encodedVideoPath !== encodedVideoPath) {
await this.storageRepository.moveFile(asset.encodedVideoPath, encodedVideoPath);
await this.assetRepository.save({ id: asset.id, encodedVideoPath });
}
}
await this.storageCore.moveAssetFile(asset, AssetPathType.JPEG_THUMBNAIL);
await this.storageCore.moveAssetFile(asset, AssetPathType.WEBP_THUMBNAIL);
await this.storageCore.moveAssetFile(asset, AssetPathType.ENCODED_VIDEO);
return true;
}
@ -146,15 +136,33 @@ export class MediaService {
return true;
}
async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
let path;
private async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
const { thumbnail, ffmpeg } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const path =
format === 'jpeg' ? this.storageCore.getLargeThumbnailPath(asset) : this.storageCore.getSmallThumbnailPath(asset);
this.storageCore.ensureFolders(path);
switch (asset.type) {
case AssetType.IMAGE:
path = await this.generateImageThumbnail(asset, format);
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : thumbnail.colorspace;
const thumbnailOptions = { format, size, colorspace, quality: thumbnail.quality };
await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions);
break;
case AssetType.VIDEO:
path = await this.generateVideoThumbnail(asset, format);
const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
const mainVideoStream = this.getMainStream(videoStreams);
if (!mainVideoStream) {
this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`);
return;
}
const mainAudioStream = this.getMainStream(audioStreams);
const config = { ...ffmpeg, targetResolution: size.toString() };
const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, path, options);
break;
default:
throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`);
}
@ -164,33 +172,6 @@ export class MediaService {
return path;
}
async generateImageThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
const { thumbnail } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const path = this.ensureThumbnailPath(asset, format);
const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : thumbnail.colorspace;
const thumbnailOptions = { format, size, colorspace, quality: thumbnail.quality };
await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions);
return path;
}
async generateVideoThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
const { ffmpeg, thumbnail } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
const mainVideoStream = this.getMainStream(videoStreams);
if (!mainVideoStream) {
this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`);
return;
}
const mainAudioStream = this.getMainStream(audioStreams);
const path = this.ensureThumbnailPath(asset, format);
const config = { ...ffmpeg, targetResolution: size.toString() };
const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, path, options);
return path;
}
async handleGenerateWebpThumbnail({ id }: IEntityJob) {
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset) {
@ -239,7 +220,8 @@ export class MediaService {
}
const input = asset.originalPath;
const output = this.ensureEncodedVideoPath(asset, 'mp4');
const output = this.storageCore.getEncodedVideoPath(asset);
this.storageCore.ensureFolders(output);
const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input);
const mainVideoStream = this.getMainStream(videoStreams);
@ -382,14 +364,6 @@ export class MediaService {
return handler;
}
ensureThumbnailPath(asset: AssetEntity, extension: string): string {
return this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.${extension}`);
}
ensureEncodedVideoPath(asset: AssetEntity, extension: string): string {
return this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.${extension}`);
}
isSRGB(asset: AssetEntity): boolean {
const { colorspace, profileDescription, bitsPerSample } = asset.exifInfo ?? {};
if (colorspace || profileDescription) {

20
server/src/domain/metadata/metadata.service.spec.ts
View file

@ -6,6 +6,8 @@ import {
newCryptoRepositoryMock,
newJobRepositoryMock,
newMetadataRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
} from '@test';
@ -19,6 +21,8 @@ import {
ICryptoRepository,
IJobRepository,
IMetadataRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
ImmichTags,
@ -34,6 +38,8 @@ describe(MetadataService.name, () => {
let cryptoRepository: jest.Mocked<ICryptoRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let metadataMock: jest.Mocked<IMetadataRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let sut: MetadataService;
@ -44,9 +50,21 @@ describe(MetadataService.name, () => {
cryptoRepository = newCryptoRepositoryMock();
jobMock = newJobRepositoryMock();
metadataMock = newMetadataRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
sut = new MetadataService(albumMock, assetMock, cryptoRepository, jobMock, metadataMock, storageMock, configMock);
sut = new MetadataService(
albumMock,
assetMock,
cryptoRepository,
jobMock,
metadataMock,
storageMock,
configMock,
moveMock,
personMock,
);
});
it('should be defined', () => {

10
server/src/domain/metadata/metadata.service.ts
View file

@ -12,13 +12,15 @@ import {
ICryptoRepository,
IJobRepository,
IMetadataRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
ImmichTags,
WithProperty,
WithoutProperty,
} from '../repositories';
import { StorageCore, StorageFolder } from '../storage';
import { StorageCore } from '../storage';
import { FeatureFlag, SystemConfigCore } from '../system-config';
interface DirectoryItem {
@ -73,9 +75,11 @@ export class MetadataService {
@Inject(IMetadataRepository) private repository: IMetadataRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
) {
this.storageCore = new StorageCore(storageRepository);
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
this.configCore.config$.subscribe(() => this.init());
}
@ -296,7 +300,7 @@ export class MetadataService {
localDateTime: createdAt,
checksum,
ownerId: asset.ownerId,
originalPath: this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`),
originalPath: this.storageCore.getAndroidMotionPath(asset),
originalFileName: asset.originalFileName,
isVisible: false,
isReadOnly: false,

23
server/src/domain/person/person.service.spec.ts
View file

@ -10,6 +10,7 @@ import {
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newMediaRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newSearchRepositoryMock,
newStorageRepositoryMock,
@ -23,6 +24,7 @@ import {
IJobRepository,
IMachineLearningRepository,
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
IStorageRepository,
@ -91,6 +93,7 @@ describe(PersonService.name, () => {
let jobMock: jest.Mocked<IJobRepository>;
let machineLearningMock: jest.Mocked<IMachineLearningRepository>;
let mediaMock: jest.Mocked<IMediaRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
@ -102,6 +105,7 @@ describe(PersonService.name, () => {
configMock = newSystemConfigRepositoryMock();
jobMock = newJobRepositoryMock();
machineLearningMock = newMachineLearningRepositoryMock();
moveMock = newMoveRepositoryMock();
mediaMock = newMediaRepositoryMock();
personMock = newPersonRepositoryMock();
searchMock = newSearchRepositoryMock();
@ -110,6 +114,7 @@ describe(PersonService.name, () => {
accessMock,
assetMock,
machineLearningMock,
moveMock,
mediaMock,
personMock,
searchMock,
@ -547,19 +552,19 @@ describe(PersonService.name, () => {
it('should generate a thumbnail', async () => {
personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.middle.assetId });
personMock.getFacesByIds.mockResolvedValue([faceStub.middle]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(assetMock.getByIds).toHaveBeenCalledWith([faceStub.middle.assetId]);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/pe/rs');
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/admin_id/pe/rs');
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/admin-id/thumbs/path.jpg', {
left: 95,
top: 95,
width: 110,
height: 110,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
@ -567,7 +572,7 @@ describe(PersonService.name, () => {
});
expect(personMock.update).toHaveBeenCalledWith({
id: 'person-1',
thumbnailPath: 'upload/thumbs/user-id/pe/rs/person-1.jpeg',
thumbnailPath: 'upload/thumbs/admin_id/pe/rs/person-1.jpeg',
});
});
@ -584,7 +589,7 @@ describe(PersonService.name, () => {
width: 510,
height: 510,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
@ -595,17 +600,17 @@ describe(PersonService.name, () => {
it('should generate a thumbnail without overflowing', async () => {
personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.end.assetId });
personMock.getFacesByIds.mockResolvedValue([faceStub.end]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
await sut.handleGeneratePersonThumbnail({ id: 'person-1' });
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/admin-id/thumbs/path.jpg', {
left: 297,
top: 297,
width: 202,
height: 202,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,

17
server/src/domain/person/person.service.ts
View file

@ -1,4 +1,5 @@
import { PersonEntity } from '@app/infra/entities';
import { PersonPathType } from '@app/infra/entities/move.entity';
import { BadRequestException, Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { AccessCore, Permission } from '../access';
import { AssetResponseDto, BulkIdErrorReason, BulkIdResponseDto, mapAsset } from '../asset';
@ -15,6 +16,7 @@ import {
IJobRepository,
IMachineLearningRepository,
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
IStorageRepository,
@ -23,7 +25,7 @@ import {
UpdateFacesData,
WithoutProperty,
} from '../repositories';
import { StorageCore, StorageFolder } from '../storage';
import { StorageCore } from '../storage';
import { SystemConfigCore } from '../system-config';
import {
MergePersonDto,
@ -46,6 +48,7 @@ export class PersonService {
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IMachineLearningRepository) private machineLearningRepository: IMachineLearningRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IPersonRepository) private repository: IPersonRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
@ -54,8 +57,8 @@ export class PersonService {
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {
this.access = new AccessCore(accessRepository);
this.storageCore = new StorageCore(storageRepository);
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, repository);
}
async getAll(authUser: AuthUserDto, dto: PersonSearchDto): Promise<PeopleResponseDto> {
@ -268,11 +271,7 @@ export class PersonService {
return false;
}
const path = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, person.ownerId, `${id}.jpeg`);
if (person.thumbnailPath && person.thumbnailPath !== path) {
await this.storageRepository.moveFile(person.thumbnailPath, path);
await this.repository.update({ id, thumbnailPath: path });
}
await this.storageCore.movePersonFile(person, PersonPathType.FACE);
return true;
}
@ -310,8 +309,8 @@ export class PersonService {
}
this.logger.verbose(`Cropping face for person: ${personId}`);
const thumbnailPath = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${personId}.jpeg`);
const thumbnailPath = this.storageCore.getPersonThumbnailPath(person);
this.storageCore.ensureFolders(thumbnailPath);
const halfWidth = (x2 - x1) / 2;
const halfHeight = (y2 - y1) / 2;

1
server/src/domain/repositories/index.ts
View file

@ -10,6 +10,7 @@ export * from './library.repository';
export * from './machine-learning.repository';
export * from './media.repository';
export * from './metadata.repository';
export * from './move.repository';
export * from './partner.repository';
export * from './person.repository';
export * from './search.repository';

12
server/src/domain/repositories/move.repository.ts Normal file
View file

@ -0,0 +1,12 @@
import { MoveEntity, PathType } from '@app/infra/entities';
export const IMoveRepository = 'IMoveRepository';
export type MoveCreate = Pick<MoveEntity, 'oldPath' | 'newPath' | 'entityId' | 'pathType'> & Partial<MoveEntity>;
export interface IMoveRepository {
create(entity: MoveCreate): Promise<MoveEntity>;
getByEntity(entityId: string, pathType: PathType): Promise<MoveEntity | null>;
update(entity: Partial<MoveEntity>): Promise<MoveEntity>;
delete(move: MoveEntity): Promise<MoveEntity>;
}

26
server/src/domain/server-info/server-info.service.spec.ts
View file

@ -1,20 +1,40 @@
import { newStorageRepositoryMock, newSystemConfigRepositoryMock, newUserRepositoryMock } from '@test';
import {
newAssetRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
newUserRepositoryMock,
} from '@test';
import { serverVersion } from '../domain.constant';
import { IStorageRepository, ISystemConfigRepository, IUserRepository } from '../repositories';
import {
IAssetRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
IUserRepository,
} from '../repositories';
import { ServerInfoService } from './server-info.service';
describe(ServerInfoService.name, () => {
let sut: ServerInfoService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let userMock: jest.Mocked<IUserRepository>;
beforeEach(() => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
userMock = newUserRepositoryMock();
sut = new ServerInfoService(configMock, userMock, storageMock);
sut = new ServerInfoService(assetMock, configMock, moveMock, personMock, userMock, storageMock);
});
it('should work', () => {

15
server/src/domain/server-info/server-info.service.ts
View file

@ -1,7 +1,15 @@
import { Inject, Injectable } from '@nestjs/common';
import { mimeTypes, serverVersion } from '../domain.constant';
import { asHumanReadable } from '../domain.util';
import { IStorageRepository, ISystemConfigRepository, IUserRepository, UserStatsQueryResponse } from '../repositories';
import {
IAssetRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
IUserRepository,
UserStatsQueryResponse,
} from '../repositories';
import { StorageCore, StorageFolder } from '../storage';
import { SystemConfigCore } from '../system-config';
import {
@ -20,12 +28,15 @@ export class ServerInfoService {
private storageCore: StorageCore;
constructor(
@Inject(IAssetRepository) assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IUserRepository) private userRepository: IUserRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.configCore = SystemConfigCore.create(configRepository);
this.storageCore = new StorageCore(storageRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
}
async getInfo(): Promise<ServerInfoResponseDto> {

67
server/src/domain/storage-template/storage-template.service.spec.ts
View file

@ -1,13 +1,23 @@
import { AssetPathType } from '@app/infra/entities';
import {
assetStub,
newAssetRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
newUserRepositoryMock,
userStub,
} from '@test';
import { when } from 'jest-when';
import { IAssetRepository, IStorageRepository, ISystemConfigRepository, IUserRepository } from '../repositories';
import {
IAssetRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
IUserRepository,
} from '../repositories';
import { defaults } from '../system-config/system-config.core';
import { StorageTemplateService } from './storage-template.service';
@ -15,6 +25,8 @@ describe(StorageTemplateService.name, () => {
let sut: StorageTemplateService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let userMock: jest.Mocked<IUserRepository>;
@ -25,10 +37,12 @@ describe(StorageTemplateService.name, () => {
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
userMock = newUserRepositoryMock();
sut = new StorageTemplateService(assetMock, configMock, defaults, storageMock, userMock);
sut = new StorageTemplateService(assetMock, configMock, defaults, moveMock, personMock, storageMock, userMock);
});
describe('handleMigrationSingle', () => {
@ -86,6 +100,13 @@ describe(StorageTemplateService.name, () => {
});
assetMock.save.mockResolvedValue(assetStub.image);
userMock.getList.mockResolvedValue([userStub.user1]);
moveMock.create.mockResolvedValue({
id: '123',
entityId: assetStub.image.id,
pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath,
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',
});
when(storageMock.checkFileExists)
.calledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg')
@ -153,6 +174,13 @@ describe(StorageTemplateService.name, () => {
});
assetMock.save.mockResolvedValue(assetStub.image);
userMock.getList.mockResolvedValue([userStub.user1]);
moveMock.create.mockResolvedValue({
id: '123',
entityId: assetStub.image.id,
pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath,
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
});
await sut.handleMigration();
@ -174,6 +202,13 @@ describe(StorageTemplateService.name, () => {
});
assetMock.save.mockResolvedValue(assetStub.image);
userMock.getList.mockResolvedValue([userStub.storageLabel]);
moveMock.create.mockResolvedValue({
id: '123',
entityId: assetStub.image.id,
pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath,
newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
});
await sut.handleMigration();
@ -194,6 +229,13 @@ describe(StorageTemplateService.name, () => {
hasNextPage: false,
});
storageMock.moveFile.mockRejectedValue(new Error('Read only system'));
moveMock.create.mockResolvedValue({
id: 'move-123',
entityId: '123',
pathType: AssetPathType.ORIGINAL,
oldPath: assetStub.image.originalPath,
newPath: '',
});
userMock.getList.mockResolvedValue([userStub.user1]);
await sut.handleMigration();
@ -206,27 +248,6 @@ describe(StorageTemplateService.name, () => {
expect(assetMock.save).not.toHaveBeenCalled();
});
it('should move the asset back if the database fails', async () => {
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
assetMock.save.mockRejectedValue('Connection Error!');
userMock.getList.mockResolvedValue([userStub.user1]);
await sut.handleMigration();
expect(assetMock.getAll).toHaveBeenCalled();
expect(assetMock.save).toHaveBeenCalledWith({
id: assetStub.image.id,
originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
});
expect(storageMock.moveFile.mock.calls).toEqual([
['/original/path.jpg', 'upload/library/user-id/2023/2023-02-23/asset-id.jpg'],
['upload/library/user-id/2023/2023-02-23/asset-id.jpg', '/original/path.jpg'],
]);
});
it('should not move read-only asset', async () => {
assetMock.getAll.mockResolvedValue({
items: [

67
server/src/domain/storage-template/storage-template.service.ts
View file

@ -1,4 +1,4 @@
import { AssetEntity, AssetType, SystemConfig } from '@app/infra/entities';
import { AssetEntity, AssetPathType, AssetType, SystemConfig } from '@app/infra/entities';
import { Inject, Injectable, Logger } from '@nestjs/common';
import handlebar from 'handlebars';
import * as luxon from 'luxon';
@ -6,7 +6,14 @@ import path from 'node:path';
import sanitize from 'sanitize-filename';
import { getLivePhotoMotionFilename, usePagination } from '../domain.util';
import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE } from '../job';
import { IAssetRepository, IStorageRepository, ISystemConfigRepository, IUserRepository } from '../repositories';
import {
IAssetRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
ISystemConfigRepository,
IUserRepository,
} from '../repositories';
import { StorageCore, StorageFolder } from '../storage';
import {
INITIAL_SYSTEM_CONFIG,
@ -36,6 +43,8 @@ export class StorageTemplateService {
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IUserRepository) private userRepository: IUserRepository,
) {
@ -43,7 +52,7 @@ export class StorageTemplateService {
this.configCore = SystemConfigCore.create(configRepository);
this.configCore.addValidator((config) => this.validate(config));
this.configCore.config$.subscribe((config) => this.onConfig(config));
this.storageCore = new StorageCore(storageRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
}
async handleMigrationSingle({ id }: IEntityJob) {
@ -90,51 +99,29 @@ export class StorageTemplateService {
}
async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
if (asset.isReadOnly || asset.isExternal) {
if (asset.isReadOnly || asset.isExternal || this.storageCore.isAndroidMotionPath(asset.originalPath)) {
// External assets are not affected by storage template
// TODO: shouldn't this only apply to external assets?
return;
}
const destination = await this.getTemplatePath(asset, metadata);
if (asset.originalPath !== destination) {
const source = asset.originalPath;
const { id, sidecarPath, originalPath } = asset;
const oldPath = originalPath;
const newPath = await this.getTemplatePath(asset, metadata);
let sidecarMoved = false;
try {
await this.storageRepository.moveFile(asset.originalPath, destination);
let sidecarDestination;
try {
if (asset.sidecarPath) {
sidecarDestination = `${destination}.xmp`;
await this.storageRepository.moveFile(asset.sidecarPath, sidecarDestination);
sidecarMoved = true;
}
await this.assetRepository.save({ id: asset.id, originalPath: destination, sidecarPath: sidecarDestination });
asset.originalPath = destination;
asset.sidecarPath = sidecarDestination || null;
} catch (error: any) {
this.logger.warn(
`Unable to save new originalPath to database, undoing move for path ${asset.originalPath} - filename ${asset.originalFileName} - id ${asset.id}`,
error?.stack,
);
// Either sidecar move failed or the save failed. Either way, move media back
await this.storageRepository.moveFile(destination, source);
if (asset.sidecarPath && sidecarDestination && sidecarMoved) {
// If the sidecar was moved, that means the saved failed. So move both the sidecar and the
// media back into their original positions
await this.storageRepository.moveFile(sidecarDestination, asset.sidecarPath);
}
}
} catch (error: any) {
this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination });
try {
await this.storageCore.moveFile({ entityId: id, pathType: AssetPathType.ORIGINAL, oldPath, newPath });
if (sidecarPath) {
await this.storageCore.moveFile({
entityId: id,
pathType: AssetPathType.SIDECAR,
oldPath: sidecarPath,
newPath: `${newPath}.xmp`,
});
}
} catch (error: any) {
this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, oldPath, newPath });
}
return asset;
}
private async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise<string> {

153
server/src/domain/storage/storage.core.ts
View file

@ -1,6 +1,8 @@
import { join } from 'node:path';
import { AssetEntity, AssetPathType, PathType, PersonEntity, PersonPathType } from '@app/infra/entities';
import { Logger } from '@nestjs/common';
import { dirname, join } from 'node:path';
import { APP_MEDIA_LOCATION } from '../domain.constant';
import { IStorageRepository } from '../repositories';
import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories';
export enum StorageFolder {
ENCODED_VIDEO = 'encoded-video',
@ -10,13 +12,26 @@ export enum StorageFolder {
THUMBNAILS = 'thumbs',
}
export class StorageCore {
constructor(private repository: IStorageRepository) {}
export interface MoveRequest {
entityId: string;
pathType: PathType;
oldPath: string | null;
newPath: string;
}
getFolderLocation(
folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
userId: string,
) {
type GeneratedAssetPath = AssetPathType.JPEG_THUMBNAIL | AssetPathType.WEBP_THUMBNAIL | AssetPathType.ENCODED_VIDEO;
export class StorageCore {
private logger = new Logger(StorageCore.name);
constructor(
private repository: IStorageRepository,
private assetRepository: IAssetRepository,
private moveRepository: IMoveRepository,
private personRepository: IPersonRepository,
) {}
getFolderLocation(folder: StorageFolder, userId: string) {
return join(this.getBaseFolder(folder), userId);
}
@ -28,21 +43,119 @@ export class StorageCore {
return join(APP_MEDIA_LOCATION, folder);
}
ensurePath(
folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
ownerId: string,
fileName: string,
): string {
const folderPath = join(
this.getFolderLocation(folder, ownerId),
fileName.substring(0, 2),
fileName.substring(2, 4),
);
this.repository.mkdirSync(folderPath);
return join(folderPath, fileName);
getPersonThumbnailPath(person: PersonEntity) {
return this.getNestedPath(StorageFolder.THUMBNAILS, person.ownerId, `${person.id}.jpeg`);
}
getLargeThumbnailPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.jpeg`);
}
getSmallThumbnailPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.webp`);
}
getEncodedVideoPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.mp4`);
}
getAndroidMotionPath(asset: AssetEntity) {
return this.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}-MP.mp4`);
}
isAndroidMotionPath(originalPath: string) {
return originalPath.startsWith(this.getBaseFolder(StorageFolder.ENCODED_VIDEO));
}
async moveAssetFile(asset: AssetEntity, pathType: GeneratedAssetPath) {
const { id: entityId, resizePath, webpPath, encodedVideoPath } = asset;
switch (pathType) {
case AssetPathType.JPEG_THUMBNAIL:
return this.moveFile({ entityId, pathType, oldPath: resizePath, newPath: this.getLargeThumbnailPath(asset) });
case AssetPathType.WEBP_THUMBNAIL:
return this.moveFile({ entityId, pathType, oldPath: webpPath, newPath: this.getSmallThumbnailPath(asset) });
case AssetPathType.ENCODED_VIDEO:
return this.moveFile({
entityId,
pathType,
oldPath: encodedVideoPath,
newPath: this.getEncodedVideoPath(asset),
});
}
}
async movePersonFile(person: PersonEntity, pathType: PersonPathType) {
const { id: entityId, thumbnailPath } = person;
switch (pathType) {
case PersonPathType.FACE:
await this.moveFile({
entityId,
pathType,
oldPath: thumbnailPath,
newPath: this.getPersonThumbnailPath(person),
});
}
}
async moveFile(request: MoveRequest) {
const { entityId, pathType, oldPath, newPath } = request;
if (!oldPath || oldPath === newPath) {
return;
}
this.ensureFolders(newPath);
let move = await this.moveRepository.getByEntity(entityId, pathType);
if (move) {
this.logger.log(`Attempting to finish incomplete move: ${move.oldPath} => ${move.newPath}`);
const oldPathExists = await this.repository.checkFileExists(move.oldPath);
const newPathExists = await this.repository.checkFileExists(move.newPath);
const actualPath = newPathExists ? move.newPath : oldPathExists ? move.oldPath : null;
if (!actualPath) {
this.logger.warn('Unable to complete move. File does not exist at either location.');
return;
}
this.logger.log(`Found file at ${actualPath === move.oldPath ? 'old' : 'new'} location`);
move = await this.moveRepository.update({ id: move.id, oldPath: actualPath, newPath });
} else {
move = await this.moveRepository.create({ entityId, pathType, oldPath, newPath });
}
if (move.oldPath !== newPath) {
await this.repository.moveFile(move.oldPath, newPath);
}
await this.savePath(pathType, entityId, newPath);
await this.moveRepository.delete(move);
}
ensureFolders(input: string) {
this.repository.mkdirSync(dirname(input));
}
removeEmptyDirs(folder: StorageFolder) {
return this.repository.removeEmptyDirs(this.getBaseFolder(folder));
}
private savePath(pathType: PathType, id: string, newPath: string) {
switch (pathType) {
case AssetPathType.ORIGINAL:
return this.assetRepository.save({ id, originalPath: newPath });
case AssetPathType.JPEG_THUMBNAIL:
return this.assetRepository.save({ id, resizePath: newPath });
case AssetPathType.WEBP_THUMBNAIL:
return this.assetRepository.save({ id, webpPath: newPath });
case AssetPathType.ENCODED_VIDEO:
return this.assetRepository.save({ id, encodedVideoPath: newPath });
case AssetPathType.SIDECAR:
return this.assetRepository.save({ id, sidecarPath: newPath });
case PersonPathType.FACE:
return this.personRepository.update({ id, thumbnailPath: newPath });
}
}
private getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
return join(this.getFolderLocation(folder, ownerId), filename.substring(0, 2), filename.substring(2, 4), filename);
}
}

17
server/src/domain/storage/storage.service.spec.ts
View file

@ -1,14 +1,25 @@
import { newStorageRepositoryMock } from '@test';
import { IStorageRepository } from '../repositories';
import {
newAssetRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
} from '@test';
import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories';
import { StorageService } from './storage.service';
describe(StorageService.name, () => {
let sut: StorageService;
let assetMock: jest.Mocked<IAssetRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
sut = new StorageService(storageMock);
sut = new StorageService(assetMock, moveMock, personMock, storageMock);
});
it('should work', () => {

11
server/src/domain/storage/storage.service.ts
View file

@ -1,6 +1,6 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
import { IDeleteFilesJob } from '../job';
import { IStorageRepository } from '../repositories';
import { IAssetRepository, IMoveRepository, IPersonRepository, IStorageRepository } from '../repositories';
import { StorageCore, StorageFolder } from './storage.core';
@Injectable()
@ -8,8 +8,13 @@ export class StorageService {
private logger = new Logger(StorageService.name);
private storageCore: StorageCore;
constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {
this.storageCore = new StorageCore(storageRepository);
constructor(
@Inject(IAssetRepository) assetRepository: IAssetRepository,
@Inject(IMoveRepository) private moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
) {
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
}
init() {

22
server/src/domain/user/user.service.spec.ts
View file

@ -11,6 +11,8 @@ import {
newCryptoRepositoryMock,
newJobRepositoryMock,
newLibraryRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
newUserRepositoryMock,
userStub,
@ -24,6 +26,8 @@ import {
ICryptoRepository,
IJobRepository,
ILibraryRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
IUserRepository,
} from '../repositories';
@ -135,18 +139,32 @@ describe(UserService.name, () => {
let assetMock: jest.Mocked<IAssetRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let libraryMock: jest.Mocked<ILibraryRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
beforeEach(async () => {
cryptoRepositoryMock = newCryptoRepositoryMock();
albumMock = newAlbumRepositoryMock();
assetMock = newAssetRepositoryMock();
cryptoRepositoryMock = newCryptoRepositoryMock();
jobMock = newJobRepositoryMock();
libraryMock = newLibraryRepositoryMock();
moveMock = newMoveRepositoryMock();
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
userMock = newUserRepositoryMock();
sut = new UserService(userMock, cryptoRepositoryMock, libraryMock, albumMock, assetMock, jobMock, storageMock);
sut = new UserService(
albumMock,
assetMock,
cryptoRepositoryMock,
jobMock,
libraryMock,
moveMock,
personMock,
storageMock,
userMock,
);
when(userMock.get).calledWith(adminUser.id).mockResolvedValue(adminUser);
when(userMock.get).calledWith(adminUser.id, undefined).mockResolvedValue(adminUser);

12
server/src/domain/user/user.service.ts
View file

@ -10,6 +10,8 @@ import {
ICryptoRepository,
IJobRepository,
ILibraryRepository,
IMoveRepository,
IPersonRepository,
IStorageRepository,
IUserRepository,
} from '../repositories';
@ -32,15 +34,17 @@ export class UserService {
private userCore: UserCore;
constructor(
@Inject(IUserRepository) private userRepository: IUserRepository,
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@Inject(ILibraryRepository) libraryRepository: ILibraryRepository,
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ICryptoRepository) cryptoRepository: ICryptoRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ILibraryRepository) libraryRepository: ILibraryRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IUserRepository) private userRepository: IUserRepository,
) {
this.storageCore = new StorageCore(storageRepository);
this.storageCore = new StorageCore(storageRepository, assetRepository, moveRepository, personRepository);
this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository);
}

3
server/src/infra/entities/index.ts
View file

@ -5,6 +5,7 @@ import { AssetEntity } from './asset.entity';
import { AuditEntity } from './audit.entity';
import { ExifEntity } from './exif.entity';
import { LibraryEntity } from './library.entity';
import { MoveEntity } from './move.entity';
import { PartnerEntity } from './partner.entity';
import { PersonEntity } from './person.entity';
import { SharedLinkEntity } from './shared-link.entity';
@ -21,6 +22,7 @@ export * from './asset.entity';
export * from './audit.entity';
export * from './exif.entity';
export * from './library.entity';
export * from './move.entity';
export * from './partner.entity';
export * from './person.entity';
export * from './shared-link.entity';
@ -37,6 +39,7 @@ export const databaseEntities = [
AssetFaceEntity,
AuditEntity,
ExifEntity,
MoveEntity,
PartnerEntity,
PersonEntity,
SharedLinkEntity,

37
server/src/infra/entities/move.entity.ts Normal file
View file

@ -0,0 +1,37 @@
import { Column, Entity, PrimaryGeneratedColumn, Unique } from 'typeorm';
@Entity('move_history')
// path lock (per entity)
@Unique('UQ_entityId_pathType', ['entityId', 'pathType'])
// new path lock (global)
@Unique('UQ_newPath', ['newPath'])
export class MoveEntity {
@PrimaryGeneratedColumn('uuid')
id!: string;
@Column({ type: 'varchar' })
entityId!: string;
@Column({ type: 'varchar' })
pathType!: PathType;
@Column({ type: 'varchar' })
oldPath!: string;
@Column({ type: 'varchar' })
newPath!: string;
}
export enum AssetPathType {
ORIGINAL = 'original',
JPEG_THUMBNAIL = 'jpeg_thumbnail',
WEBP_THUMBNAIL = 'webp_thumbnail',
ENCODED_VIDEO = 'encoded_video',
SIDECAR = 'sidecar',
}
export enum PersonPathType {
FACE = 'face',
}
export type PathType = AssetPathType | PersonPathType;

3
server/src/infra/infra.module.ts
View file

@ -11,6 +11,7 @@ import {
IMachineLearningRepository,
IMediaRepository,
IMetadataRepository,
IMoveRepository,
IPartnerRepository,
IPersonRepository,
ISearchRepository,
@ -44,6 +45,7 @@ import {
MachineLearningRepository,
MediaRepository,
MetadataRepository,
MoveRepository,
PartnerRepository,
PersonRepository,
SharedLinkRepository,
@ -67,6 +69,7 @@ const providers: Provider[] = [
{ provide: IKeyRepository, useClass: APIKeyRepository },
{ provide: IMachineLearningRepository, useClass: MachineLearningRepository },
{ provide: IMetadataRepository, useClass: MetadataRepository },
{ provide: IMoveRepository, useClass: MoveRepository },
{ provide: IPartnerRepository, useClass: PartnerRepository },
{ provide: IPersonRepository, useClass: PersonRepository },
{ provide: ISearchRepository, useClass: TypesenseRepository },

14
server/src/infra/migrations/1696968880063-AddMoveTable.ts Normal file
View file

@ -0,0 +1,14 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddMoveTable1696968880063 implements MigrationInterface {
name = 'AddMoveTable1696968880063'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE TABLE "move_history" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "entityId" character varying NOT NULL, "pathType" character varying NOT NULL, "oldPath" character varying NOT NULL, "newPath" character varying NOT NULL, CONSTRAINT "UQ_newPath" UNIQUE ("newPath"), CONSTRAINT "UQ_entityId_pathType" UNIQUE ("entityId", "pathType"), CONSTRAINT "PK_af608f132233acf123f2949678d" PRIMARY KEY ("id"))`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP TABLE "move_history"`);
}
}

5
server/src/infra/repositories/filesystem.provider.ts
View file

@ -6,6 +6,7 @@ import {
IStorageRepository,
mimeTypes,
} from '@app/domain';
import { Logger } from '@nestjs/common';
import archiver from 'archiver';
import { constants, createReadStream, existsSync, mkdirSync } from 'fs';
import fs, { readdir, writeFile } from 'fs/promises';
@ -17,6 +18,8 @@ import path from 'path';
const moveFile = promisify<string, string, mv.Options>(mv);
export class FilesystemProvider implements IStorageRepository {
private logger = new Logger(FilesystemProvider.name);
createZipStream(): ImmichZipStream {
const archive = archiver('zip', { store: true });
@ -52,6 +55,8 @@ export class FilesystemProvider implements IStorageRepository {
writeFile = writeFile;
async moveFile(source: string, destination: string): Promise<void> {
this.logger.verbose(`Moving ${source} to ${destination}`);
if (await this.checkFileExists(destination)) {
throw new Error(`Destination file already exists: ${destination}`);
}

1
server/src/infra/repositories/index.ts
View file

@ -11,6 +11,7 @@ export * from './library.repository';
export * from './machine-learning.repository';
export * from './media.repository';
export * from './metadata.repository';
export * from './move.repository';
export * from './partner.repository';
export * from './person.repository';
export * from './shared-link.repository';

2
server/src/infra/repositories/job.repository.ts
View file

@ -70,6 +70,8 @@ export class JobRepository implements IJobRepository {
private getJobOptions(item: JobItem): JobsOptions | null {
switch (item.name) {
case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE:
return { jobId: item.data.id };
case JobName.GENERATE_PERSON_THUMBNAIL:
return { priority: 1 };

26
server/src/infra/repositories/move.repository.ts Normal file
View file

@ -0,0 +1,26 @@
import { IMoveRepository, MoveCreate } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { MoveEntity, PathType } from '../entities';
@Injectable()
export class MoveRepository implements IMoveRepository {
constructor(@InjectRepository(MoveEntity) private repository: Repository<MoveEntity>) {}
create(entity: MoveCreate): Promise<MoveEntity> {
return this.repository.save(entity);
}
getByEntity(entityId: string, pathType: PathType): Promise<MoveEntity | null> {
return this.repository.findOne({ where: { entityId, pathType } });
}
update(entity: Partial<MoveEntity>): Promise<MoveEntity> {
return this.repository.save(entity);
}
delete(move: MoveEntity): Promise<MoveEntity> {
return this.repository.remove(move);
}
}

39
server/test/fixtures/asset.stub.ts vendored
View file

@ -116,6 +116,45 @@ export const assetStub = {
sidecarPath: null,
deletedAt: null,
}),
primaryImage: Object.freeze<AssetEntity>({
id: 'asset-id',
deviceAssetId: 'device-asset-id',
fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'),
fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'),
owner: userStub.admin,
ownerId: 'admin-id',
deviceId: 'device-id',
originalPath: '/original/path.jpg',
resizePath: '/uploads/admin-id/thumbs/path.jpg',
checksum: Buffer.from('file hash', 'utf8'),
type: AssetType.IMAGE,
webpPath: '/uploads/admin-id/webp/path.ext',
thumbhash: Buffer.from('blablabla', 'base64'),
encodedVideoPath: null,
createdAt: new Date('2023-02-23T05:06:29.716Z'),
updatedAt: new Date('2023-02-23T05:06:29.716Z'),
localDateTime: new Date('2023-02-23T05:06:29.716Z'),
isFavorite: true,
isArchived: false,
isReadOnly: false,
duration: null,
isVisible: true,
isExternal: false,
livePhotoVideo: null,
livePhotoVideoId: null,
isOffline: false,
libraryId: 'library-id',
library: libraryStub.uploadLibrary1,
tags: [],
sharedLinks: [],
originalFileName: 'asset-id.jpg',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5_000,
} as ExifEntity,
}),
image: Object.freeze<AssetEntity>({
id: 'asset-id',
deviceAssetId: 'device-asset-id',

1
server/test/repositories/index.ts
View file

@ -10,6 +10,7 @@ export * from './library.repository.mock';
export * from './machine-learning.repository.mock';
export * from './media.repository.mock';
export * from './metadata.repository.mock';
export * from './move.repository.mock';
export * from './partner.repository.mock';
export * from './person.repository.mock';
export * from './search.repository.mock';

10
server/test/repositories/move.repository.mock.ts Normal file
View file

@ -0,0 +1,10 @@
import { IMoveRepository } from '@app/domain';
export const newMoveRepositoryMock = (): jest.Mocked<IMoveRepository> => {
return {
create: jest.fn(),
getByEntity: jest.fn(),
update: jest.fn(),
delete: jest.fn(),
};
};