JPEG + temp wip

desktop/src/main/ipc.ts

@@ -44,8 +44,11 @@ import { getDirFiles } from "./services/fs";
 import {
     convertToJPEG,
     generateImageThumbnail,
-} from "./services/imageProcessor";
-import { clipImageEmbedding, clipTextEmbeddingIfAvailable } from "./services/ml-clip";
+} from "./services/convert";
+import {
+    clipImageEmbedding,
+    clipTextEmbeddingIfAvailable,
+} from "./services/ml-clip";
 import { detectFaces, faceEmbedding } from "./services/ml-face";
 import {
     clearStores,
@@ -142,8 +145,8 @@ export const attachIPCHandlers = () => {
 
     // - Conversion
 
-    ipcMain.handle("convertToJPEG", (_, fileData, filename) =>
-        convertToJPEG(fileData, filename),
+    ipcMain.handle("convertToJPEG", (_, fileName, imageData) =>
+        convertToJPEG(fileName, imageData),
     );
 
     ipcMain.handle(

desktop/src/main/services/imageProcessor.ts → desktop/src/main/services/convert.ts

@@ -1,12 +1,11 @@
 import { existsSync } from "fs";
 import fs from "node:fs/promises";
 import path from "path";
-import { CustomErrors, ElectronFile } from "../../types/ipc";
+import { CustomErrorMessage, ElectronFile } from "../../types/ipc";
 import log from "../log";
 import { writeStream } from "../stream";
-import { generateTempFilePath } from "../temp";
+import { deleteTempFile, generateTempFilePath } from "../temp";
 import { execAsync, isDev } from "../utils-electron";
-import { deleteTempFile } from "./ffmpeg";
 
 const IMAGE_MAGICK_PLACEHOLDER = "IMAGE_MAGICK";
 const MAX_DIMENSION_PLACEHOLDER = "MAX_DIMENSION";
@@ -69,27 +68,20 @@ const IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [
 const imageMagickStaticPath = () =>
     path.join(isDev ? "build" : process.resourcesPath, "image-magick");
 
-export async function convertToJPEG(
-    fileData: Uint8Array,
-    filename: string,
-): Promise<Uint8Array> {
+export const convertToJPEG = async (
+    fileName: string,
+    imageData: Uint8Array,
+): Promise<Uint8Array> => {
     if (process.platform == "win32")
-        throw Error(CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED);
-    const convertedFileData = await convertToJPEG_(fileData, filename);
-    return convertedFileData;
-}
+        throw new Error(CustomErrorMessage.NotAvailable);
 
-async function convertToJPEG_(
-    fileData: Uint8Array,
-    filename: string,
-): Promise<Uint8Array> {
     let tempInputFilePath: string;
     let tempOutputFilePath: string;
     try {
-        tempInputFilePath = await generateTempFilePath(filename);
+        tempInputFilePath = await generateTempFilePath(fileName);
         tempOutputFilePath = await generateTempFilePath("output.jpeg");
 
-        await fs.writeFile(tempInputFilePath, fileData);
+        await fs.writeFile(tempInputFilePath, imageData);
 
         await execAsync(
             constructConvertCommand(tempInputFilePath, tempOutputFilePath),
@@ -114,7 +106,7 @@ async function convertToJPEG_(
             );
         }
     }
-}
+};
 
 function constructConvertCommand(
     tempInputFilePath: string,

desktop/src/main/services/ffmpeg.ts

@@ -4,9 +4,9 @@ import fs from "node:fs/promises";
 import { ElectronFile } from "../../types/ipc";
 import log from "../log";
 import { writeStream } from "../stream";
-import { generateTempFilePath, getTempDirPath } from "../temp";
 import { withTimeout } from "../utils";
 import { execAsync } from "../utils-electron";
+import { deleteTempFile, generateTempFilePath } from "../utils-temp";
 
 const INPUT_PATH_PLACEHOLDER = "INPUT";
 const FFMPEG_PLACEHOLDER = "FFMPEG";
@@ -120,16 +120,3 @@ const ffmpegBinaryPath = () => {
     // https://github.com/eugeneware/ffmpeg-static/issues/16
     return pathToFfmpeg.replace("app.asar", "app.asar.unpacked");
 };
-
-export async function writeTempFile(fileStream: Uint8Array, fileName: string) {
-    const tempFilePath = await generateTempFilePath(fileName);
-    await fs.writeFile(tempFilePath, fileStream);
-    return tempFilePath;
-}
-
-export async function deleteTempFile(tempFilePath: string) {
-    const tempDirPath = await getTempDirPath();
-    if (!tempFilePath.startsWith(tempDirPath))
-        log.error("Attempting to delete a non-temp file ${tempFilePath}");
-    await fs.rm(tempFilePath, { force: true });
-}

desktop/src/main/services/ml-clip.ts

@@ -11,8 +11,7 @@ import * as ort from "onnxruntime-node";
 import Tokenizer from "../../thirdparty/clip-bpe-ts/mod";
 import log from "../log";
 import { writeStream } from "../stream";
-import { generateTempFilePath } from "../temp";
-import { deleteTempFile } from "./ffmpeg";
+import { deleteTempFile, generateTempFilePath } from "../utils-temp";
 import { makeCachedInferenceSession } from "./ml";
 
 const cachedCLIPImageSession = makeCachedInferenceSession(

desktop/src/main/temp.ts

@@ -1,35 +0,0 @@
-import { app } from "electron/main";
-import { existsSync } from "node:fs";
-import fs from "node:fs/promises";
-import path from "path";
-
-const CHARACTERS =
-    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
-
-export async function getTempDirPath() {
-    const tempDirPath = path.join(app.getPath("temp"), "ente");
-    await fs.mkdir(tempDirPath, { recursive: true });
-    return tempDirPath;
-}
-
-function generateTempName(length: number) {
-    let result = "";
-
-    const charactersLength = CHARACTERS.length;
-    for (let i = 0; i < length; i++) {
-        result += CHARACTERS.charAt(
-            Math.floor(Math.random() * charactersLength),
-        );
-    }
-    return result;
-}
-
-export async function generateTempFilePath(formatSuffix: string) {
-    let tempFilePath: string;
-    do {
-        const tempDirPath = await getTempDirPath();
-        const namePrefix = generateTempName(10);
-        tempFilePath = path.join(tempDirPath, namePrefix + "-" + formatSuffix);
-    } while (existsSync(tempFilePath));
-    return tempFilePath;
-}

desktop/src/main/utils-temp.ts

@@ -0,0 +1,64 @@
+import { app } from "electron/main";
+import { existsSync } from "node:fs";
+import fs from "node:fs/promises";
+import path from "path";
+
+/**
+ * Our very own directory within the system temp directory. Go crazy, but
+ * remember to clean up, especially in exception handlers.
+ */
+const enteTempDirPath = async () => {
+    const result = path.join(app.getPath("temp"), "ente");
+    await fs.mkdir(result, { recursive: true });
+    return result;
+};
+
+const randomPrefix = (length: number) => {
+    const CHARACTERS =
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+
+    let result = "";
+    const charactersLength = CHARACTERS.length;
+    for (let i = 0; i < length; i++) {
+        result += CHARACTERS.charAt(
+            Math.floor(Math.random() * charactersLength),
+        );
+    }
+    return result;
+};
+
+/**
+ * Return the path to a temporary file with the given {@link formatSuffix}.
+ *
+ * The function returns the path to a file in the system temp directory (in an
+ * Ente specific folder therin) with a random prefix and the given
+ * {@link formatSuffix}. It ensures that there is no existing file with the same
+ * name already.
+ *
+ * Use {@link deleteTempFile} to remove this file when you're done.
+ */
+export const generateTempFilePath = async (formatSuffix: string) => {
+    const tempDir = await enteTempDirPath();
+    let result: string;
+    do {
+        result = path.join(tempDir, randomPrefix(10) + "-" + formatSuffix);
+    } while (existsSync(result));
+    return result;
+};
+
+/**
+ * Delete a temporary file at the given path if it exists.
+ *
+ * This is the same as a vanilla {@link fs.rm}, except it first checks that the
+ * given path is within the Ente specific directory in the system temp
+ * directory. This acts as an additional safety check.
+ *
+ * @param tempFilePath The path to the temporary file to delete. This path
+ * should've been previously created using {@link generateTempFilePath}.
+ */
+export const deleteTempFile = async (tempFilePath: string) => {
+    const tempDir = await enteTempDirPath();
+    if (!tempFilePath.startsWith(tempDir))
+        throw new Error(`Attempting to delete a non-temp file ${tempFilePath}`);
+    await fs.rm(tempFilePath, { force: true });
+};

desktop/src/preload.ts

@@ -127,10 +127,10 @@ const fsIsDir = (dirPath: string): Promise<boolean> =>
 // - Conversion
 
 const convertToJPEG = (
-    fileData: Uint8Array,
-    filename: string,
+    fileName: string,
+    imageData: Uint8Array,
 ): Promise<Uint8Array> =>
-    ipcRenderer.invoke("convertToJPEG", fileData, filename);
+    ipcRenderer.invoke("convertToJPEG", fileName, imageData);
 
 const generateImageThumbnail = (
     inputFile: File | ElectronFile,

web/apps/photos/src/utils/file/index.ts

@@ -1,5 +1,4 @@
 import { decodeLivePhoto } from "@/media/live-photo";
-import { convertBytesToHumanReadable } from "@/next/file";
 import log from "@/next/log";
 import { CustomErrorMessage, type Electron } from "@/next/types/ipc";
 import { workerBridge } from "@/next/worker/worker-bridge";
@@ -68,7 +67,7 @@ class ModuleState {
      *
      * Note the double negative when it is used.
      */
-    isElectronJPEGConversionNotAvailable = false;
+    isNativeJPEGConversionNotAvailable = false;
 }
 
 const moduleState = new ModuleState();
@@ -282,7 +281,10 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => {
     try {
         const tempFile = new File([imageBlob], fileName);
         fileTypeInfo = await getFileType(tempFile);
-        log.debug(() => `Obtaining renderable image for ${JSON.stringify(fileTypeInfo)}`);
+        log.debug(
+            () =>
+                `Obtaining renderable image for ${JSON.stringify(fileTypeInfo)}`,
+        );
         const { exactType } = fileTypeInfo;
 
         if (!isRawFile(exactType)) {
@@ -292,18 +294,15 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => {
 
         let jpegBlob: Blob | undefined;
 
-        const available = !moduleState.isElectronJPEGConversionNotAvailable;
+        const available = !moduleState.isNativeJPEGConversionNotAvailable;
         if (isElectron() && available && isSupportedRawFormat(exactType)) {
             // If we're running in our desktop app, see if our Node.js layer can
             // convert this into a JPEG using native tools for us.
             try {
-                jpegBlob = await tryConvertToJPEGInElectron(
-                    imageBlob,
-                    fileName,
-                );
+                jpegBlob = await nativeConvertToJPEG(fileName, imageBlob);
             } catch (e) {
                 if (e.message == CustomErrorMessage.NotAvailable) {
-                    moduleState.isElectronJPEGConversionNotAvailable = true;
+                    moduleState.isNativeJPEGConversionNotAvailable = true;
                 } else {
                     log.error("Native conversion to JPEG failed", e);
                     throw e;
@@ -326,27 +325,18 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => {
     }
 };
 
-const tryConvertToJPEGInElectron = async (
-    fileBlob: Blob,
-    filename: string,
-): Promise<Blob | undefined> => {
+const nativeConvertToJPEG = async (fileName: string, imageBlob: Blob) => {
     const startTime = Date.now();
-    const inputFileData = new Uint8Array(await fileBlob.arrayBuffer());
+    const imageData = new Uint8Array(await imageBlob.arrayBuffer());
     const electron = globalThis.electron;
     // If we're running in a worker, we need to reroute the request back to
     // the main thread since workers don't have access to the `window` (and
     // thus, to the `window.electron`) object.
-    const convertedFileData = electron
-        ? await electron.convertToJPEG(inputFileData, filename)
-        : await workerBridge.convertToJPEG(inputFileData, filename);
-    log.info(
-        `originalFileSize:${convertBytesToHumanReadable(
-            fileBlob?.size,
-        )},convertedFileSize:${convertBytesToHumanReadable(
-            convertedFileData?.length,
-        )},  native conversion time: ${Date.now() - startTime}ms `,
-    );
-    return new Blob([convertedFileData]);
+    const jpegData = electron
+        ? await electron.convertToJPEG(fileName, imageData)
+        : await workerBridge.convertToJPEG(fileName, imageData);
+    log.info(`Native JPEG conversion took ${Date.now() - startTime} ms`);
+    return new Blob([jpegData]);
 };
 
 export function isFileHEIC(exactType: string) {

web/packages/next/types/ipc.ts

@@ -191,26 +191,26 @@ export interface Electron {
         isDir: (dirPath: string) => Promise<boolean>;
     };
 
-    /*
-     * TODO: AUDIT below this - Some of the types we use below are not copyable
-     * across process boundaries, and such functions will (expectedly) fail at
-     * runtime. For such functions, find an efficient alternative or refactor
-     * the dataflow.
-     */
-
     // - Conversion
 
     /**
-     * Try to convert an arbitrary image into JPEG.
+     * Try to convert an arbitrary image into JPEG using native layer tools.
+     *
+     * The behaviour is OS dependent. On macOS we use the `sips` utility, and on
+     * some Linux architectures we use an ImageMagick binary bundled with our
+     * desktop app.
      *
-     * The behaviour is OS dependent.
-     * @param fileData
-     * @param filename
-     * @returns
+     * In other cases (primarily Windows), where native JPEG conversion is not
+     * yet possible, this method will throw an error with the
+     * {@link CustomErrorMessage.NotAvailable} message..
+     *
+     * @param fileName The name of the file whose data we're being given.
+     * @param imageData The raw image data (the contents of the image file).
+     * @returns JPEG data.
      */
     convertToJPEG: (
-        fileData: Uint8Array,
-        filename: string,
+        fileName: string,
+        imageData: Uint8Array,
     ) => Promise<Uint8Array>;
 
     generateImageThumbnail: (
@@ -439,6 +439,13 @@ export interface Electron {
         filePaths: string[],
     ) => Promise<void>;
 
+    /*
+     * TODO: AUDIT below this - Some of the types we use below are not copyable
+     * across process boundaries, and such functions will (expectedly) fail at
+     * runtime. For such functions, find an efficient alternative or refactor
+     * the dataflow.
+     */
+
     // -
 
     getElectronFilesFromGoogleZip: (

web/packages/next/worker/comlink-worker.ts

@@ -43,15 +43,16 @@ export class ComlinkWorker<T extends new () => InstanceType<T>> {
  * `workerBridge` object after importing it from `worker-bridge.ts`.
  *
  * Not all workers need access to all these functions, and this can indeed be
- * done in a more fine-grained, per-worker, manner if needed.
+ * done in a more fine-grained, per-worker, manner if needed. For now, since it
+ * is a motley bunch, we just inject them all.
  */
 const workerBridge = {
     // Needed: generally (presumably)
     logToDisk,
     // Needed by ML worker
     getAuthToken: () => ensureLocalUser().then((user) => user.token),
-    convertToJPEG: (inputFileData: Uint8Array, filename: string) =>
-        ensureElectron().convertToJPEG(inputFileData, filename),
+    convertToJPEG: (fileName: string, imageData: Uint8Array) =>
+        ensureElectron().convertToJPEG(fileName, imageData),
     detectFaces: (input: Float32Array) => ensureElectron().detectFaces(input),
     faceEmbedding: (input: Float32Array) =>
         ensureElectron().faceEmbedding(input),