feat(web,server): add thumbhash support (#2649)

* add thumbhash: server generation and web impl

* move logic to infra & use byta in db

* remove unnecesary logs

* update generated API and simplify thumbhash gen

* fix check errors

* removed unnecessary library and css tag

* style edits

* syntax mistake

* update server test, change thumbhash job name

* fix tests

* Update server/src/domain/asset/response-dto/asset-response.dto.ts

Co-authored-by: Thomas <9749173+uhthomas@users.noreply.github.com>

* add unit test, change migration date

* change to official thumbhash impl

* update call method to not use eval

* "generate missing" looks for thumbhash

* improve queue & improve syntax

* update syntax again

* update tests

* fix thumbhash generation

* consolidate queueing to avoid duplication

* cover all types of incorrect thumbnail cases

* split out jest tasks

* put back thumbnail duration loading for images without thumbhash

* Remove stray package.json

---------

Co-authored-by: Luke McCarthy <mail@lukehmcc.com>
Co-authored-by: Thomas <9749173+uhthomas@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

mobile/openapi/doc/AssetResponseDto.md

@@ -16,6 +16,7 @@ Name | Type | Description | Notes
 **originalPath** | **String** |  | 
 **originalFileName** | **String** |  | 
 **resized** | **bool** |  | 
+**thumbhash** | **String** | base64 encoded thumbhash | 
 **fileCreatedAt** | [**DateTime**](DateTime.md) |  | 
 **fileModifiedAt** | [**DateTime**](DateTime.md) |  | 
 **updatedAt** | [**DateTime**](DateTime.md) |  | 

mobile/openapi/lib/model/asset_response_dto.dart

@@ -21,6 +21,7 @@ class AssetResponseDto {
     required this.originalPath,
     required this.originalFileName,
     required this.resized,
+    required this.thumbhash,
     required this.fileCreatedAt,
     required this.fileModifiedAt,
     required this.updatedAt,
@@ -52,6 +53,9 @@ class AssetResponseDto {
 
   bool resized;
 
+  /// base64 encoded thumbhash
+  String? thumbhash;
+
   DateTime fileCreatedAt;
 
   DateTime fileModifiedAt;
@@ -101,6 +105,7 @@ class AssetResponseDto {
      other.originalPath == originalPath &&
      other.originalFileName == originalFileName &&
      other.resized == resized &&
+     other.thumbhash == thumbhash &&
      other.fileCreatedAt == fileCreatedAt &&
      other.fileModifiedAt == fileModifiedAt &&
      other.updatedAt == updatedAt &&
@@ -126,6 +131,7 @@ class AssetResponseDto {
     (originalPath.hashCode) +
     (originalFileName.hashCode) +
     (resized.hashCode) +
+    (thumbhash == null ? 0 : thumbhash!.hashCode) +
     (fileCreatedAt.hashCode) +
     (fileModifiedAt.hashCode) +
     (updatedAt.hashCode) +
@@ -141,7 +147,7 @@ class AssetResponseDto {
     (checksum.hashCode);
 
   @override
-  String toString() => 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, originalFileName=$originalFileName, resized=$resized, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, isArchived=$isArchived, mimeType=$mimeType, duration=$duration, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags, people=$people, checksum=$checksum]';
+  String toString() => 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, originalFileName=$originalFileName, resized=$resized, thumbhash=$thumbhash, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, isArchived=$isArchived, mimeType=$mimeType, duration=$duration, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags, people=$people, checksum=$checksum]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -153,6 +159,11 @@ class AssetResponseDto {
       json[r'originalPath'] = this.originalPath;
       json[r'originalFileName'] = this.originalFileName;
       json[r'resized'] = this.resized;
+    if (this.thumbhash != null) {
+      json[r'thumbhash'] = this.thumbhash;
+    } else {
+      // json[r'thumbhash'] = null;
+    }
       json[r'fileCreatedAt'] = this.fileCreatedAt.toUtc().toIso8601String();
       json[r'fileModifiedAt'] = this.fileModifiedAt.toUtc().toIso8601String();
       json[r'updatedAt'] = this.updatedAt.toUtc().toIso8601String();
@@ -212,6 +223,7 @@ class AssetResponseDto {
         originalPath: mapValueOfType<String>(json, r'originalPath')!,
         originalFileName: mapValueOfType<String>(json, r'originalFileName')!,
         resized: mapValueOfType<bool>(json, r'resized')!,
+        thumbhash: mapValueOfType<String>(json, r'thumbhash'),
         fileCreatedAt: mapDateTime(json, r'fileCreatedAt', '')!,
         fileModifiedAt: mapDateTime(json, r'fileModifiedAt', '')!,
         updatedAt: mapDateTime(json, r'updatedAt', '')!,
@@ -280,6 +292,7 @@ class AssetResponseDto {
     'originalPath',
     'originalFileName',
     'resized',
+    'thumbhash',
     'fileCreatedAt',
     'fileModifiedAt',
     'updatedAt',

mobile/openapi/test/asset_response_dto_test.dart

@@ -56,6 +56,12 @@ void main() {
       // TODO
     });
 
+    // base64 encoded thumbhash
+    // String thumbhash
+    test('to test the property `thumbhash`', () async {
+      // TODO
+    });
+
     // DateTime fileCreatedAt
     test('to test the property `fileCreatedAt`', () async {
       // TODO

server/immich-openapi-specs.json

@@ -4865,6 +4865,11 @@
           "resized": {
             "type": "boolean"
           },
+          "thumbhash": {
+            "type": "string",
+            "nullable": true,
+            "description": "base64 encoded thumbhash"
+          },
           "fileCreatedAt": {
             "format": "date-time",
             "type": "string"
@@ -4926,6 +4931,7 @@
           "originalPath",
           "originalFileName",
           "resized",
+          "thumbhash",
           "fileCreatedAt",
           "fileModifiedAt",
           "updatedAt",

server/package-lock.json

@@ -46,6 +46,7 @@
         "rxjs": "^7.2.0",
         "sanitize-filename": "^1.6.3",
         "sharp": "^0.31.3",
+        "thumbhash": "^0.1.1",
         "typeorm": "^0.3.11",
         "typesense": "^1.5.3",
         "ua-parser-js": "^1.0.35"
@@ -4234,9 +4235,9 @@
       }
     },
     "node_modules/bullmq": {
-      "version": "3.14.1",
-      "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz",
-      "integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==",
+      "version": "3.15.4",
+      "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.4.tgz",
+      "integrity": "sha512-jig63/PWODJEsAuswiCVUHaDWMv5fGpU36SjI0watAdXZMmy9K/iMKQAfPXmfZeK98bY/+co/efaDWVh3eVImw==",
       "dependencies": {
         "cron-parser": "^4.6.0",
         "glob": "^8.0.3",
@@ -10806,6 +10807,11 @@
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="
     },
+    "node_modules/thumbhash": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz",
+      "integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg=="
+    },
     "node_modules/tmp": {
       "version": "0.0.33",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
@@ -15241,9 +15247,9 @@
       "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="
     },
     "bullmq": {
-      "version": "3.14.1",
-      "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz",
-      "integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==",
+      "version": "3.15.4",
+      "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.15.4.tgz",
+      "integrity": "sha512-jig63/PWODJEsAuswiCVUHaDWMv5fGpU36SjI0watAdXZMmy9K/iMKQAfPXmfZeK98bY/+co/efaDWVh3eVImw==",
       "requires": {
         "cron-parser": "^4.6.0",
         "glob": "^8.0.3",
@@ -20185,6 +20191,11 @@
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="
     },
+    "thumbhash": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz",
+      "integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg=="
+    },
     "tmp": {
       "version": "0.0.33",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",

server/package.json

@@ -75,6 +75,7 @@
     "rxjs": "^7.2.0",
     "sanitize-filename": "^1.6.3",
     "sharp": "^0.31.3",
+    "thumbhash": "^0.1.1",
     "typeorm": "^0.3.11",
     "typesense": "^1.5.3",
     "ua-parser-js": "^1.0.35"

server/src/domain/asset/response-dto/asset-response.dto.ts

@@ -16,6 +16,8 @@ export class AssetResponseDto {
   originalPath!: string;
   originalFileName!: string;
   resized!: boolean;
+  /**base64 encoded thumbhash */
+  thumbhash!: string | null;
   fileCreatedAt!: Date;
   fileModifiedAt!: Date;
   updatedAt!: Date;
@@ -42,6 +44,7 @@ export function mapAsset(entity: AssetEntity): AssetResponseDto {
     originalPath: entity.originalPath,
     originalFileName: entity.originalFileName,
     resized: !!entity.resizePath,
+    thumbhash: entity.thumbhash?.toString('base64') ?? null,
     fileCreatedAt: entity.fileCreatedAt,
     fileModifiedAt: entity.fileModifiedAt,
     updatedAt: entity.updatedAt,
@@ -68,6 +71,7 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto {
     originalPath: entity.originalPath,
     originalFileName: entity.originalFileName,
     resized: !!entity.resizePath,
+    thumbhash: entity.thumbhash?.toString('base64') || null,
     fileCreatedAt: entity.fileCreatedAt,
     fileModifiedAt: entity.fileModifiedAt,
     updatedAt: entity.updatedAt,

server/src/domain/job/job.constants.ts

@@ -27,6 +27,7 @@ export enum JobName {
   QUEUE_GENERATE_THUMBNAILS = 'queue-generate-thumbnails',
   GENERATE_JPEG_THUMBNAIL = 'generate-jpeg-thumbnail',
   GENERATE_WEBP_THUMBNAIL = 'generate-webp-thumbnail',
+  GENERATE_THUMBHASH_THUMBNAIL = 'generate-thumbhash-thumbnail',
 
   // metadata
   QUEUE_METADATA_EXTRACTION = 'queue-metadata-extraction',
@@ -92,6 +93,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.QUEUE_GENERATE_THUMBNAILS]: QueueName.THUMBNAIL_GENERATION,
   [JobName.GENERATE_JPEG_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
   [JobName.GENERATE_WEBP_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
+  [JobName.GENERATE_THUMBHASH_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION,
 
   // metadata
   [JobName.QUEUE_METADATA_EXTRACTION]: QueueName.METADATA_EXTRACTION,

server/src/domain/job/job.repository.ts

@@ -31,6 +31,7 @@ export type JobItem =
   | { name: JobName.QUEUE_GENERATE_THUMBNAILS; data: IBaseJob }
   | { name: JobName.GENERATE_JPEG_THUMBNAIL; data: IEntityJob }
   | { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IEntityJob }
+  | { name: JobName.GENERATE_THUMBHASH_THUMBNAIL; data: IEntityJob }
 
   // User Deletion
   | { name: JobName.USER_DELETE_CHECK; data?: IBaseJob }

server/src/domain/job/job.service.spec.ts

@@ -261,7 +261,13 @@ describe(JobService.name, () => {
       },
       {
         item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
-        jobs: [JobName.GENERATE_WEBP_THUMBNAIL, JobName.CLASSIFY_IMAGE, JobName.ENCODE_CLIP, JobName.RECOGNIZE_FACES],
+        jobs: [
+          JobName.GENERATE_WEBP_THUMBNAIL,
+          JobName.CLASSIFY_IMAGE,
+          JobName.ENCODE_CLIP,
+          JobName.RECOGNIZE_FACES,
+          JobName.GENERATE_THUMBHASH_THUMBNAIL,
+        ],
       },
       {
         item: { name: JobName.CLASSIFY_IMAGE, data: { id: 'asset-1' } },

server/src/domain/job/job.service.ts

@@ -160,6 +160,7 @@ export class JobService {
 
       case JobName.GENERATE_JPEG_THUMBNAIL: {
         await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: item.data });
+        await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: item.data });
         await this.jobRepository.queue({ name: JobName.CLASSIFY_IMAGE, data: item.data });
         await this.jobRepository.queue({ name: JobName.ENCODE_CLIP, data: item.data });
         await this.jobRepository.queue({ name: JobName.RECOGNIZE_FACES, data: item.data });

server/src/domain/media/media.repository.ts

@@ -47,6 +47,7 @@ export interface IMediaRepository {
   // image
   resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void>;
   crop(input: string, options: CropOptions): Promise<Buffer>;
+  generateThumbhash(imagePath: string): Promise<Buffer>;
 
   // video
   extractVideoThumbnail(input: string, output: string, size: number): Promise<void>;

server/src/domain/media/media.service.spec.ts

@@ -54,9 +54,9 @@ describe(MediaService.name, () => {
       });
     });
 
-    it('should queue all assets with missing thumbnails', async () => {
+    it('should queue all assets with missing resize path', async () => {
       assetMock.getWithout.mockResolvedValue({
-        items: [assetEntityStub.image],
+        items: [assetEntityStub.noResizePath],
         hasNextPage: false,
       });
 
@@ -69,6 +69,38 @@ describe(MediaService.name, () => {
         data: { id: assetEntityStub.image.id },
       });
     });
+
+    it('should queue all assets with missing webp path', async () => {
+      assetMock.getWithout.mockResolvedValue({
+        items: [assetEntityStub.noWebpPath],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueGenerateThumbnails({ force: false });
+
+      expect(assetMock.getAll).not.toHaveBeenCalled();
+      expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_WEBP_THUMBNAIL,
+        data: { id: assetEntityStub.image.id },
+      });
+    });
+
+    it('should queue all assets with missing thumbhash', async () => {
+      assetMock.getWithout.mockResolvedValue({
+        items: [assetEntityStub.noThumbhash],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueGenerateThumbnails({ force: false });
+
+      expect(assetMock.getAll).not.toHaveBeenCalled();
+      expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
+      expect(jobMock.queue).toHaveBeenCalledWith({
+        name: JobName.GENERATE_THUMBHASH_THUMBNAIL,
+        data: { id: assetEntityStub.image.id },
+      });
+    });
   });
 
   describe('handleGenerateJpegThumbnail', () => {
@@ -129,6 +161,25 @@ describe(MediaService.name, () => {
     });
   });
 
+  describe('handleGenerateThumbhashThumbnail', () => {
+    it('should skip thumbhash generation if resize path is missing', async () => {
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath]);
+      await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.noResizePath.id });
+      expect(mediaMock.generateThumbhash).not.toHaveBeenCalled();
+    });
+
+    it('should generate a thumbhash', async () => {
+      const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8');
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
+      mediaMock.generateThumbhash.mockResolvedValue(thumbhashBuffer);
+
+      await sut.handleGenerateThumbhashThumbnail({ id: assetEntityStub.image.id });
+
+      expect(mediaMock.generateThumbhash).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.ext');
+      expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', thumbhash: thumbhashBuffer });
+    });
+  });
+
   describe('handleQueueVideoConversion', () => {
     it('should queue all video assets', async () => {
       assetMock.getAll.mockResolvedValue({

server/src/domain/media/media.service.ts

@@ -37,7 +37,16 @@ export class MediaService {
 
     for await (const assets of assetPagination) {
       for (const asset of assets) {
-        await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } });
+        if (!asset.resizePath || force) {
+          await this.jobRepository.queue({ name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: asset.id } });
+          continue;
+        }
+        if (!asset.webpPath) {
+          await this.jobRepository.queue({ name: JobName.GENERATE_WEBP_THUMBNAIL, data: { id: asset.id } });
+        }
+        if (!asset.thumbhash) {
+          await this.jobRepository.queue({ name: JobName.GENERATE_THUMBHASH_THUMBNAIL, data: { id: asset.id } });
+        }
       }
     }
 
@@ -87,6 +96,18 @@ export class MediaService {
     return true;
   }
 
+  async handleGenerateThumbhashThumbnail({ id }: IEntityJob): Promise<boolean> {
+    const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset?.resizePath) {
+      return false;
+    }
+
+    const thumbhash = await this.mediaRepository.generateThumbhash(asset.resizePath);
+    await this.assetRepository.save({ id: asset.id, thumbhash });
+
+    return true;
+  }
+
   async handleQueueVideoConversion(job: IBaseJob) {
     const { force } = job;
 

server/src/immich/api-v1/asset/asset.core.ts

@@ -35,6 +35,7 @@ export class AssetCore {
       livePhotoVideo: livePhotoAssetId != null ? ({ id: livePhotoAssetId } as AssetEntity) : null,
       resizePath: null,
       webpPath: null,
+      thumbhash: null,
       encodedVideoPath: null,
       tags: [],
       sharedLinks: [],

server/src/infra/entities/asset.entity.ts

@@ -51,6 +51,9 @@ export class AssetEntity {
   @Column({ type: 'varchar', nullable: true, default: '' })
   webpPath!: string | null;
 
+  @Column({ type: 'bytea', nullable: true })
+  thumbhash!: Buffer | null;
+
   @Column({ type: 'varchar', nullable: true, default: '' })
   encodedVideoPath!: string | null;
 

server/src/infra/migrations/1686762895180-AddThumbhashColumn.ts

@@ -0,0 +1,13 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddThumbhashColumn1685546571785 implements MigrationInterface {
+  name = 'AddThumbhashColumn1686762895180';
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "assets" ADD "thumbhash" bytea NULL`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE "assets" DROP COLUMN "thumbhash"`);
+  }
+}

server/src/infra/repositories/asset.repository.ts

@@ -135,6 +135,7 @@ export class AssetRepository implements IAssetRepository {
           { resizePath: '', isVisible: true },
           { webpPath: IsNull(), isVisible: true },
           { webpPath: '', isVisible: true },
+          { thumbhash: IsNull(), isVisible: true },
         ];
         break;
 

server/src/infra/repositories/media.repository.ts

@@ -119,4 +119,17 @@ export class MediaRepository implements IMediaRepository {
         .run();
     });
   }
+
+  async generateThumbhash(imagePath: string): Promise<Buffer> {
+    const maxSize = 100;
+
+    const { data, info } = await sharp(imagePath)
+      .resize(maxSize, maxSize, { fit: 'inside', withoutEnlargement: true })
+      .raw()
+      .ensureAlpha()
+      .toBuffer({ resolveWithObject: true });
+
+    const thumbhash = await import('thumbhash');
+    return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data));
+  }
 }

server/src/microservices/app.service.ts

@@ -61,6 +61,7 @@ export class AppService {
       [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
       [JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
       [JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
+      [JobName.GENERATE_THUMBHASH_THUMBNAIL]: (data) => this.mediaService.handleGenerateThumbhashThumbnail(data),
       [JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
       [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
       [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),

server/test/fixtures.ts

@@ -196,7 +196,37 @@ export const assetEntityStub = {
     resizePath: null,
     checksum: Buffer.from('file hash', 'utf8'),
     type: AssetType.IMAGE,
+    webpPath: '/uploads/user-id/webp/path.ext',
+    thumbhash: Buffer.from('blablabla', 'base64'),
+    encodedVideoPath: null,
+    createdAt: new Date('2023-02-23T05:06:29.716Z'),
+    updatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    mimeType: null,
+    isFavorite: true,
+    isArchived: false,
+    duration: null,
+    isVisible: true,
+    livePhotoVideo: null,
+    livePhotoVideoId: null,
+    tags: [],
+    sharedLinks: [],
+    faces: [],
+    sidecarPath: null,
+  }),
+  noWebpPath: Object.freeze<AssetEntity>({
+    id: 'asset-id',
+    deviceAssetId: 'device-asset-id',
+    fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'),
+    fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    owner: userEntityStub.user1,
+    ownerId: 'user-id',
+    deviceId: 'device-id',
+    originalPath: '/original/path.ext',
+    resizePath: '/uploads/user-id/thumbs/path.ext',
+    checksum: Buffer.from('file hash', 'utf8'),
+    type: AssetType.IMAGE,
     webpPath: null,
+    thumbhash: Buffer.from('blablabla', 'base64'),
     encodedVideoPath: null,
     createdAt: new Date('2023-02-23T05:06:29.716Z'),
     updatedAt: new Date('2023-02-23T05:06:29.716Z'),
@@ -209,6 +239,37 @@ export const assetEntityStub = {
     livePhotoVideoId: null,
     tags: [],
     sharedLinks: [],
+    originalFileName: 'asset-id.ext',
+    faces: [],
+    sidecarPath: null,
+  }),
+  noThumbhash: Object.freeze<AssetEntity>({
+    id: 'asset-id',
+    deviceAssetId: 'device-asset-id',
+    fileModifiedAt: new Date('2023-02-23T05:06:29.716Z'),
+    fileCreatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    owner: userEntityStub.user1,
+    ownerId: 'user-id',
+    deviceId: 'device-id',
+    originalPath: '/original/path.ext',
+    resizePath: '/uploads/user-id/thumbs/path.ext',
+    checksum: Buffer.from('file hash', 'utf8'),
+    type: AssetType.IMAGE,
+    webpPath: '/uploads/user-id/webp/path.ext',
+    thumbhash: null,
+    encodedVideoPath: null,
+    createdAt: new Date('2023-02-23T05:06:29.716Z'),
+    updatedAt: new Date('2023-02-23T05:06:29.716Z'),
+    mimeType: null,
+    isFavorite: true,
+    isArchived: false,
+    duration: null,
+    isVisible: true,
+    livePhotoVideo: null,
+    livePhotoVideoId: null,
+    tags: [],
+    sharedLinks: [],
+    originalFileName: 'asset-id.ext',
     faces: [],
     sidecarPath: null,
   }),
@@ -224,7 +285,8 @@ export const assetEntityStub = {
     resizePath: '/uploads/user-id/thumbs/path.ext',
     checksum: Buffer.from('file hash', 'utf8'),
     type: AssetType.IMAGE,
-    webpPath: null,
+    webpPath: '/uploads/user-id/webp/path.ext',
+    thumbhash: Buffer.from('blablabla', 'base64'),
     encodedVideoPath: null,
     createdAt: new Date('2023-02-23T05:06:29.716Z'),
     updatedAt: new Date('2023-02-23T05:06:29.716Z'),
@@ -255,6 +317,7 @@ export const assetEntityStub = {
     checksum: Buffer.from('file hash', 'utf8'),
     type: AssetType.VIDEO,
     webpPath: null,
+    thumbhash: null,
     encodedVideoPath: null,
     createdAt: new Date('2023-02-23T05:06:29.716Z'),
     updatedAt: new Date('2023-02-23T05:06:29.716Z'),
@@ -305,6 +368,7 @@ export const assetEntityStub = {
     sidecarPath: null,
     type: AssetType.IMAGE,
     webpPath: null,
+    thumbhash: null,
     encodedVideoPath: null,
     createdAt: new Date('2023-02-23T05:06:29.716Z'),
     updatedAt: new Date('2023-02-23T05:06:29.716Z'),
@@ -334,6 +398,7 @@ export const assetEntityStub = {
     deviceId: 'device-id',
     originalPath: '/original/path.ext',
     resizePath: '/uploads/user-id/thumbs/path.ext',
+    thumbhash: null,
     checksum: Buffer.from('file hash', 'utf8'),
     type: AssetType.IMAGE,
     webpPath: null,
@@ -507,6 +572,7 @@ const assetResponse: AssetResponseDto = {
   originalPath: 'fake_path/jpeg',
   originalFileName: 'asset_1.jpeg',
   resized: false,
+  thumbhash: null,
   fileModifiedAt: today,
   fileCreatedAt: today,
   updatedAt: today,
@@ -787,6 +853,7 @@ export const sharedLinkStub = {
             clipEmbedding: [0.12, 0.13, 0.14],
           },
           webpPath: '',
+          thumbhash: null,
           encodedVideoPath: '',
           duration: null,
           isVisible: true,

server/test/repositories/media.repository.mock.ts

@@ -3,6 +3,7 @@ import { IMediaRepository } from '@app/domain';
 export const newMediaRepositoryMock = (): jest.Mocked<IMediaRepository> => {
   return {
     extractVideoThumbnail: jest.fn(),
+    generateThumbhash: jest.fn(),
     resize: jest.fn(),
     crop: jest.fn(),
     probe: jest.fn(),

server/tsconfig.json

@@ -9,6 +9,7 @@
     "allowSyntheticDefaultImports": true,
     "resolveJsonModule": true,
     "target": "es2017",
+    "moduleResolution": "node16",
     "sourceMap": true,
     "outDir": "./dist",
     "incremental": true,

web/package-lock.json

@@ -20,7 +20,8 @@
 				"rxjs": "^7.8.0",
 				"socket.io-client": "^4.6.1",
 				"svelte-local-storage-store": "^0.5.0",
-				"svelte-material-icons": "^3.0.4"
+				"svelte-material-icons": "^3.0.4",
+				"unlazy": "^0.8.9"
 			},
 			"devDependencies": {
 				"@babel/preset-env": "^7.20.2",
@@ -4134,6 +4135,15 @@
 				"url": "https://opencollective.com/typescript-eslint"
 			}
 		},
+		"node_modules/@unlazy/core": {
+			"version": "0.8.9",
+			"resolved": "https://registry.npmjs.org/@unlazy/core/-/core-0.8.9.tgz",
+			"integrity": "sha512-DQ4WB/cuEWTknU/59uRwpSipvMAJzBDmRyaHDUc1RcXi0Z7/Vcl0EE7BpROxEynqd1EI+2oMWQaDLyXffUdUiA==",
+			"dependencies": {
+				"fast-blurhash": "^1.1.2",
+				"thumbhash": "^0.1.1"
+			}
+		},
 		"node_modules/@zoom-image/core": {
 			"version": "0.18.2",
 			"resolved": "https://registry.npmjs.org/@zoom-image/core/-/core-0.18.2.tgz",
@@ -5945,6 +5955,11 @@
 				"node": ">= 14"
 			}
 		},
+		"node_modules/fast-blurhash": {
+			"version": "1.1.2",
+			"resolved": "https://registry.npmjs.org/fast-blurhash/-/fast-blurhash-1.1.2.tgz",
+			"integrity": "sha512-lJVOgYSlahqkRhrKumNx/SGB2F/qS0D1z7xjGYjb5EZJRtlzySGMniZjkQ9h9Rv8sPmM/V9orEgRiMwazDNH6A=="
+		},
 		"node_modules/fast-deep-equal": {
 			"version": "3.1.3",
 			"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -11217,6 +11232,11 @@
 				"node": ">=0.8"
 			}
 		},
+		"node_modules/thumbhash": {
+			"version": "0.1.1",
+			"resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz",
+			"integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg=="
+		},
 		"node_modules/tiny-glob": {
 			"version": "0.2.9",
 			"resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz",
@@ -11441,6 +11461,18 @@
 				"node": ">= 4.0.0"
 			}
 		},
+		"node_modules/unlazy": {
+			"version": "0.8.9",
+			"resolved": "https://registry.npmjs.org/unlazy/-/unlazy-0.8.9.tgz",
+			"integrity": "sha512-lRCuXN20N1esqSQqtSVBLAw9GJz0lcBuOBs3UGGw7cFWHQlWJVZZ3OviwOl42f1CnVHjAON1rs2hIdJWgMAUyg==",
+			"dependencies": {
+				"@unlazy/core": "0.8.9"
+			},
+			"peerDependencies": {
+				"fast-blurhash": "^1.1.2",
+				"thumbhash": "^0.1.1"
+			}
+		},
 		"node_modules/update-browserslist-db": {
 			"version": "1.0.10",
 			"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz",
@@ -14739,6 +14771,15 @@
 				"eslint-visitor-keys": "^3.3.0"
 			}
 		},
+		"@unlazy/core": {
+			"version": "0.8.9",
+			"resolved": "https://registry.npmjs.org/@unlazy/core/-/core-0.8.9.tgz",
+			"integrity": "sha512-DQ4WB/cuEWTknU/59uRwpSipvMAJzBDmRyaHDUc1RcXi0Z7/Vcl0EE7BpROxEynqd1EI+2oMWQaDLyXffUdUiA==",
+			"requires": {
+				"fast-blurhash": "^1.1.2",
+				"thumbhash": "^0.1.1"
+			}
+		},
 		"@zoom-image/core": {
 			"version": "0.18.2",
 			"resolved": "https://registry.npmjs.org/@zoom-image/core/-/core-0.18.2.tgz",
@@ -16053,6 +16094,11 @@
 				"source-map-support": "^0.5.21"
 			}
 		},
+		"fast-blurhash": {
+			"version": "1.1.2",
+			"resolved": "https://registry.npmjs.org/fast-blurhash/-/fast-blurhash-1.1.2.tgz",
+			"integrity": "sha512-lJVOgYSlahqkRhrKumNx/SGB2F/qS0D1z7xjGYjb5EZJRtlzySGMniZjkQ9h9Rv8sPmM/V9orEgRiMwazDNH6A=="
+		},
 		"fast-deep-equal": {
 			"version": "3.1.3",
 			"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -19861,6 +19907,11 @@
 				"thenify": ">= 3.1.0 < 4"
 			}
 		},
+		"thumbhash": {
+			"version": "0.1.1",
+			"resolved": "https://registry.npmjs.org/thumbhash/-/thumbhash-0.1.1.tgz",
+			"integrity": "sha512-kH5pKeIIBPQXAOni2AiY/Cu/NKdkFREdpH+TLdM0g6WA7RriCv0kPLgP731ady67MhTAqrVG/4mnEeibVuCJcg=="
+		},
 		"tiny-glob": {
 			"version": "0.2.9",
 			"resolved": "https://registry.npmjs.org/tiny-glob/-/tiny-glob-0.2.9.tgz",
@@ -20023,6 +20074,14 @@
 			"integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
 			"dev": true
 		},
+		"unlazy": {
+			"version": "0.8.9",
+			"resolved": "https://registry.npmjs.org/unlazy/-/unlazy-0.8.9.tgz",
+			"integrity": "sha512-lRCuXN20N1esqSQqtSVBLAw9GJz0lcBuOBs3UGGw7cFWHQlWJVZZ3OviwOl42f1CnVHjAON1rs2hIdJWgMAUyg==",
+			"requires": {
+				"@unlazy/core": "0.8.9"
+			}
+		},
 		"update-browserslist-db": {
 			"version": "1.0.10",
 			"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.10.tgz",

web/package.json

@@ -70,6 +70,7 @@
 		"rxjs": "^7.8.0",
 		"socket.io-client": "^4.6.1",
 		"svelte-local-storage-store": "^0.5.0",
-		"svelte-material-icons": "^3.0.4"
+		"svelte-material-icons": "^3.0.4",
+		"unlazy": "^0.8.9"
 	}
 }

web/src/api/open-api/api.ts

@@ -637,6 +637,12 @@ export interface AssetResponseDto {
      * @memberof AssetResponseDto
      */
     'resized': boolean;
+    /**
+     * base64 encoded thumbhash
+     * @type {string}
+     * @memberof AssetResponseDto
+     */
+    'thumbhash': string | null;
     /**
      * 
      * @type {string}

web/src/lib/components/asset-viewer/detail-panel.svelte

@@ -120,6 +120,7 @@
 							altText={person.name}
 							widthStyle="90px"
 							heightStyle="90px"
+							thumbhash={null}
 						/>
 						<p class="font-medium mt-1 truncate">{person.name}</p>
 					</a>

web/src/lib/components/assets/thumbnail/image-thumbnail.svelte

@@ -1,27 +1,58 @@
 <script lang="ts">
+	import { onMount } from 'svelte';
+	import { lazyLoad } from 'unlazy';
 	import { imageLoad } from '$lib/utils/image-load';
 
 	export let url: string;
 	export let altText: string;
 	export let heightStyle: string | undefined = undefined;
 	export let widthStyle: string;
+	export let thumbhash: string | null = null;
 	export let curve = false;
 	export let shadow = false;
 	export let circle = false;
 	let loading = true;
+
+	let imageElement: HTMLImageElement;
+
+	onMount(() => {
+		if (thumbhash) {
+			lazyLoad(imageElement, {
+				hash: thumbhash,
+				hashType: 'thumbhash'
+			});
+		}
+	});
 </script>
 
-<img
-	style:width={widthStyle}
-	style:height={heightStyle}
-	src={url}
-	alt={altText}
-	class="object-cover transition-opacity duration-300"
-	class:rounded-lg={curve}
-	class:shadow-lg={shadow}
-	class:rounded-full={circle}
-	class:opacity-0={loading}
-	draggable="false"
-	use:imageLoad
-	on:image-load|once={() => (loading = false)}
-/>
+{#if thumbhash}
+	<img
+		style:width={widthStyle}
+		style:height={heightStyle}
+		data-src={url}
+		alt={altText}
+		class="object-cover"
+		class:rounded-lg={curve}
+		class:shadow-lg={shadow}
+		class:rounded-full={circle}
+		draggable="false"
+		bind:this={imageElement}
+	/>
+
+	<!-- not everthing yet has thumbhash support so the old method is kept -->
+{:else}
+	<img
+		style:width={widthStyle}
+		style:height={heightStyle}
+		src={url}
+		alt={altText}
+		class="object-cover transition-opacity duration-300"
+		class:rounded-lg={curve}
+		class:shadow-lg={shadow}
+		class:rounded-full={circle}
+		class:opacity-0={loading}
+		draggable="false"
+		use:imageLoad
+		on:image-load|once={() => (loading = false)}
+	/>
+{/if}

web/src/lib/components/assets/thumbnail/thumbnail.svelte

@@ -129,6 +129,7 @@
 						altText={asset.originalFileName}
 						widthStyle="{width}px"
 						heightStyle="{height}px"
+						thumbhash={asset.thumbhash}
 					/>
 				{:else}
 					<div class="w-full h-full p-4 flex items-center justify-center">