use imagemagick and libraw for raw image support (#2668)

* use imagemagick and libraw for raw image support

imagemagick and libraw have generally good support for raw images, including
Sony's ARW format. These tools should also allow Immich to support many more
image formats in future without any major code changes.

https://www.libraw.org/supported-cameras

I've tested and verified this change with .ARW files and other standard formats.

Fixes: #2156

* Add additional type for awr

* pr feedback

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

mobile/lib/utils/files_helper.dart

@@ -53,6 +53,9 @@ class FileHelper {
       case 'insv':
         return {"type": "video", "subType": "mp4"};
 
+      case 'arw':
+        return {"type": "image", "subType": "x-sony-arw"};
+
       default:
         return {"type": "unsupport", "subType": "unsupport"};
     }

server/Dockerfile

@@ -2,7 +2,7 @@ FROM node:18.16.0-alpine3.18@sha256:f41850f74ff16a33daff988e2ea06ef8f5daeb6fb849
 
 WORKDIR /usr/src/app
 
-RUN apk add --update-cache build-base python3 vips-heif vips-dev ffmpeg perl
+RUN apk add --update-cache build-base imagemagick-dev python3 ffmpeg libraw-dev perl vips-dev vips-heif vips-magick
 
 COPY package.json package-lock.json ./
 
@@ -23,7 +23,7 @@ ENV NODE_ENV=production
 
 WORKDIR /usr/src/app
 
-RUN apk add --no-cache vips-heif vips vips-cpp ffmpeg perl
+RUN apk add --no-cache ffmpeg imagemagick-dev libraw-dev perl vips-dev vips-heif vips-magick
 
 COPY --from=prod /usr/src/app/node_modules ./node_modules
 COPY --from=prod /usr/src/app/dist ./dist

server/package-lock.json

@@ -45,7 +45,7 @@
         "reflect-metadata": "^0.1.13",
         "rxjs": "^7.2.0",
         "sanitize-filename": "^1.6.3",
-        "sharp": "^0.31.0",
+        "sharp": "^0.31.3",
         "typeorm": "^0.3.11",
         "typesense": "^1.5.3",
         "ua-parser-js": "^1.0.35"

server/package.json

@@ -74,7 +74,7 @@
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.2.0",
     "sanitize-filename": "^1.6.3",
-    "sharp": "^0.31.0",
+    "sharp": "^0.31.3",
     "typeorm": "^0.3.11",
     "typesense": "^1.5.3",
     "ua-parser-js": "^1.0.35"

server/src/domain/media/media.service.ts

@@ -60,9 +60,9 @@ export class MediaService {
           size: JPEG_THUMBNAIL_SIZE,
           format: 'jpeg',
         });
-      } catch (error) {
+      } catch (error: any) {
         this.logger.warn(
-          `Failed to generate jpeg thumbnail using sharp, trying with exiftool-vendored (asset=${asset.id})`,
+          `Failed to generate jpeg thumbnail using sharp, trying with exiftool-vendored (asset=${asset.id}): ${error.message}`,
         );
         await this.mediaRepository.extractThumbnailFromExif(asset.originalPath, jpegThumbnailPath);
       }

server/src/immich/config/asset-upload.config.spec.ts

@@ -49,77 +49,37 @@ describe('assetUploadOption', () => {
       expect(name).toBeUndefined();
     });
 
-    it('should allow images', async () => {
-      const file = { mimetype: 'image/jpeg', originalname: 'test.jpg' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow videos', async () => {
-      const file = { mimetype: 'video/mp4', originalname: 'test.mp4' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow webm videos', async () => {
-      const file = { mimetype: 'video/webm', originalname: 'test.webm' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .raf recognized', () => {
-      const file = { mimetype: 'image/x-fuji-raf', originalname: 'test.raf' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .srw recognized', () => {
-      const file = { mimetype: 'image/x-samsung-srw', originalname: 'test.srw' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .wmv videos', () => {
-      const file = { mimetype: 'video/x-ms-wmv', originalname: 'test.wmv' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .mkv videos', () => {
-      const file = { mimetype: 'video/x-matroska', originalname: 'test.mkv' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .mpg videos', () => {
-      const file = { mimetype: 'video/mpeg', originalname: 'test.mpg' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .flv videos', () => {
-      const file = { mimetype: 'video/x-flv', originalname: 'test.flv' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .mov videos with video/mov mimetype', () => {
-      const file = { mimetype: 'video/mov', originalname: 'test.mov' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .avi videos with video/avi mimetype', () => {
-      const file = { mimetype: 'video/avi', originalname: 'test.avi' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
-
-    it('should allow .avi videos with video/x-msvideo mimetype', () => {
-      const file = { mimetype: 'video/x-msvideo', originalname: 'test.avi' } as any;
-      fileFilter(mock.userRequest, file, callback);
-      expect(callback).toHaveBeenCalledWith(null, true);
-    });
+    for (const { mimetype, extension } of [
+      { mimetype: 'image/dng', extension: 'dng' },
+      { mimetype: 'image/gif', extension: 'gif' },
+      { mimetype: 'image/heic', extension: 'heic' },
+      { mimetype: 'image/heif', extension: 'heif' },
+      { mimetype: 'image/jpeg', extension: 'jpeg' },
+      { mimetype: 'image/jpeg', extension: 'jpg' },
+      { mimetype: 'image/png', extension: 'png' },
+      { mimetype: 'image/tiff', extension: 'tiff' },
+      { mimetype: 'image/webp', extension: 'webp' },
+      { mimetype: 'image/x-adobe-dng', extension: 'dng' },
+      { mimetype: 'image/x-fuji-raf', extension: 'raf' },
+      { mimetype: 'image/x-nikon-nef', extension: 'nef' },
+      { mimetype: 'image/x-samsung-srw', extension: 'srw' },
+      { mimetype: 'image/x-sony-arw', extension: 'arw' },
+      { mimetype: 'video/avi', extension: 'avi' },
+      { mimetype: 'video/mov', extension: 'mov' },
+      { mimetype: 'video/mp4', extension: 'mp4' },
+      { mimetype: 'video/mpeg', extension: 'mpg' },
+      { mimetype: 'video/webm', extension: 'webm' },
+      { mimetype: 'video/x-flv', extension: 'flv' },
+      { mimetype: 'video/x-matroska', extension: 'mkv' },
+      { mimetype: 'video/x-ms-wmv', extension: 'wmv' },
+      { mimetype: 'video/x-msvideo', extension: 'avi' },
+    ]) {
+      const name = `test.${extension}`;
+      it(`should allow ${name} (${mimetype})`, async () => {
+        fileFilter(mock.userRequest, { mimetype, originalname: name }, callback);
+        expect(callback).toHaveBeenCalledWith(null, true);
+      });
+    }
 
     it('should not allow unknown types', async () => {
       const file = { mimetype: 'application/html', originalname: 'test.html' } as any;

server/src/immich/config/asset-upload.config.ts

@@ -55,7 +55,7 @@ function fileFilter(req: AuthRequest, file: any, cb: any) {
   }
   if (
     file.mimetype.match(
-      /\/(jpg|jpeg|png|gif|avi|mov|mp4|webm|x-msvideo|quicktime|heic|heif|dng|x-adobe-dng|webp|tiff|3gpp|nef|x-nikon-nef|x-fuji-raf|x-samsung-srw|mpeg|x-flv|x-ms-wmv|x-matroska)$/,
+      /\/(jpg|jpeg|png|gif|avi|mov|mp4|webm|x-msvideo|quicktime|heic|heif|dng|x-adobe-dng|webp|tiff|3gpp|nef|x-nikon-nef|x-fuji-raf|x-samsung-srw|mpeg|x-flv|x-ms-wmv|x-matroska|x-sony-arw|arw)$/,
     )
   ) {
     cb(null, true);

web/src/lib/utils/asset-utils.spec.ts

@@ -1,6 +1,6 @@
 import type { AssetResponseDto } from '@api';
 import { describe, expect, it } from '@jest/globals';
-import { getAssetFilename, getFilenameExtension } from './asset-utils';
+import { getAssetFilename, getFilenameExtension, getFileMimeType } from './asset-utils';
 
 describe('get file extension from filename', () => {
 	it('returns the extension without including the dot', () => {
@@ -58,3 +58,50 @@ describe('get asset filename', () => {
 		});
 	});
 });
+
+describe('get file mime type', () => {
+	for (const { extension, mimeType } of [
+		{ extension: '3gp', mimeType: 'video/3gpp' },
+		{ extension: 'arw', mimeType: 'image/x-sony-arw' },
+		{ extension: 'dng', mimeType: 'image/dng' },
+		{ extension: 'heic', mimeType: 'image/heic' },
+		{ extension: 'heif', mimeType: 'image/heif' },
+		{ extension: 'insp', mimeType: 'image/jpeg' },
+		{ extension: 'insv', mimeType: 'video/mp4' },
+		{ extension: 'nef', mimeType: 'image/x-nikon-nef' },
+		{ extension: 'raf', mimeType: 'image/x-fuji-raf' },
+		{ extension: 'srw', mimeType: 'image/x-samsung-srw' }
+	]) {
+		it(`returns the mime type for ${extension}`, () => {
+			expect(getFileMimeType({ name: `filename.${extension}` } as File)).toEqual(mimeType);
+		});
+	}
+
+	it('returns the mime type from the file', () => {
+		[
+			{
+				file: {
+					name: 'filename.jpg',
+					type: 'image/jpeg'
+				},
+				result: 'image/jpeg'
+			},
+			{
+				file: {
+					name: 'filename.txt',
+					type: 'text/plain'
+				},
+				result: 'text/plain'
+			},
+			{
+				file: {
+					name: 'filename.txt',
+					type: ''
+				},
+				result: ''
+			}
+		].forEach(({ file, result }) => {
+			expect(getFileMimeType(file as File)).toEqual(result);
+		});
+	});
+});

web/src/lib/utils/asset-utils.ts

@@ -125,34 +125,20 @@ export function getAssetFilename(asset: AssetResponseDto): string {
  * Returns the MIME type of the file and an empty string when not found.
  */
 export function getFileMimeType(file: File): string {
-	if (file.type !== '') {
-		// Return the MIME type determined by the browser.
-		return file.type;
-	}
-
-	// Return MIME type based on the file extension.
-	switch (getFilenameExtension(file.name)) {
-		case 'heic':
-			return 'image/heic';
-		case 'heif':
-			return 'image/heif';
-		case 'dng':
-			return 'image/dng';
-		case '3gp':
-			return 'video/3gpp';
-		case 'nef':
-			return 'image/x-nikon-nef';
-		case 'raf':
-			return 'image/x-fuji-raf';
-		case 'srw':
-			return 'image/x-samsung-srw';
-		case 'insp':
-			return 'image/jpeg';
-		case 'insv':
-			return 'video/mp4';
-		default:
-			return '';
-	}
+	const mimeTypes: Record<string, string> = {
+		'3gp': 'video/3gpp',
+		arw: 'image/x-sony-arw',
+		dng: 'image/dng',
+		heic: 'image/heic',
+		heif: 'image/heif',
+		insp: 'image/jpeg',
+		insv: 'video/mp4',
+		nef: 'image/x-nikon-nef',
+		raf: 'image/x-fuji-raf',
+		srw: 'image/x-samsung-srw'
+	};
+	// Return the MIME type determined by the browser or the MIME type based on the file extension.
+	return file.type || (mimeTypes[getFilenameExtension(file.name)] ?? '');
 }
 
 /**

web/src/lib/utils/file-uploader.ts

@@ -22,7 +22,7 @@ export const openFileUploadDialog = async (
 
 			// When adding a content type that is unsupported by browsers, make sure
 			// to also add it to getFileMimeType() otherwise the upload will fail.
-			fileSelector.accept = 'image/*,video/*,.heic,.heif,.dng,.3gp,.nef,.srw,.raf,.insp,.insv';
+			fileSelector.accept = 'image/*,video/*,.heic,.heif,.dng,.3gp,.nef,.srw,.raf,.insp,.insv,.arw';
 
 			fileSelector.onchange = async (e: Event) => {
 				const target = e.target as HTMLInputElement;