diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index b8763c520..3a526a01c 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -44,8 +44,11 @@ import { getDirFiles } from "./services/fs"; import { convertToJPEG, generateImageThumbnail, -} from "./services/imageProcessor"; -import { clipImageEmbedding, clipTextEmbeddingIfAvailable } from "./services/ml-clip"; +} from "./services/convert"; +import { + clipImageEmbedding, + clipTextEmbeddingIfAvailable, +} from "./services/ml-clip"; import { detectFaces, faceEmbedding } from "./services/ml-face"; import { clearStores, @@ -142,8 +145,8 @@ export const attachIPCHandlers = () => { // - Conversion - ipcMain.handle("convertToJPEG", (_, fileData, filename) => - convertToJPEG(fileData, filename), + ipcMain.handle("convertToJPEG", (_, fileName, imageData) => + convertToJPEG(fileName, imageData), ); ipcMain.handle( diff --git a/desktop/src/main/services/imageProcessor.ts b/desktop/src/main/services/convert.ts similarity index 92% rename from desktop/src/main/services/imageProcessor.ts rename to desktop/src/main/services/convert.ts index a731cc80f..b8e2c1b8d 100644 --- a/desktop/src/main/services/imageProcessor.ts +++ b/desktop/src/main/services/convert.ts @@ -1,12 +1,11 @@ import { existsSync } from "fs"; import fs from "node:fs/promises"; import path from "path"; -import { CustomErrors, ElectronFile } from "../../types/ipc"; +import { CustomErrorMessage, ElectronFile } from "../../types/ipc"; import log from "../log"; import { writeStream } from "../stream"; -import { generateTempFilePath } from "../temp"; +import { deleteTempFile, generateTempFilePath } from "../temp"; import { execAsync, isDev } from "../utils-electron"; -import { deleteTempFile } from "./ffmpeg"; const IMAGE_MAGICK_PLACEHOLDER = "IMAGE_MAGICK"; const MAX_DIMENSION_PLACEHOLDER = "MAX_DIMENSION"; @@ -69,27 +68,20 @@ const IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE = [ const imageMagickStaticPath = () => path.join(isDev ? "build" : process.resourcesPath, "image-magick"); -export async function convertToJPEG( - fileData: Uint8Array, - filename: string, -): Promise { +export const convertToJPEG = async ( + fileName: string, + imageData: Uint8Array, +): Promise => { if (process.platform == "win32") - throw Error(CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED); - const convertedFileData = await convertToJPEG_(fileData, filename); - return convertedFileData; -} + throw new Error(CustomErrorMessage.NotAvailable); -async function convertToJPEG_( - fileData: Uint8Array, - filename: string, -): Promise { let tempInputFilePath: string; let tempOutputFilePath: string; try { - tempInputFilePath = await generateTempFilePath(filename); + tempInputFilePath = await generateTempFilePath(fileName); tempOutputFilePath = await generateTempFilePath("output.jpeg"); - await fs.writeFile(tempInputFilePath, fileData); + await fs.writeFile(tempInputFilePath, imageData); await execAsync( constructConvertCommand(tempInputFilePath, tempOutputFilePath), @@ -114,7 +106,7 @@ async function convertToJPEG_( ); } } -} +}; function constructConvertCommand( tempInputFilePath: string, diff --git a/desktop/src/main/services/ffmpeg.ts b/desktop/src/main/services/ffmpeg.ts index fba43ab0f..4730bcbdc 100644 --- a/desktop/src/main/services/ffmpeg.ts +++ b/desktop/src/main/services/ffmpeg.ts @@ -4,9 +4,9 @@ import fs from "node:fs/promises"; import { ElectronFile } from "../../types/ipc"; import log from "../log"; import { writeStream } from "../stream"; -import { generateTempFilePath, getTempDirPath } from "../temp"; import { withTimeout } from "../utils"; import { execAsync } from "../utils-electron"; +import { deleteTempFile, generateTempFilePath } from "../utils-temp"; const INPUT_PATH_PLACEHOLDER = "INPUT"; const FFMPEG_PLACEHOLDER = "FFMPEG"; @@ -120,16 +120,3 @@ const ffmpegBinaryPath = () => { // https://github.com/eugeneware/ffmpeg-static/issues/16 return pathToFfmpeg.replace("app.asar", "app.asar.unpacked"); }; - -export async function writeTempFile(fileStream: Uint8Array, fileName: string) { - const tempFilePath = await generateTempFilePath(fileName); - await fs.writeFile(tempFilePath, fileStream); - return tempFilePath; -} - -export async function deleteTempFile(tempFilePath: string) { - const tempDirPath = await getTempDirPath(); - if (!tempFilePath.startsWith(tempDirPath)) - log.error("Attempting to delete a non-temp file ${tempFilePath}"); - await fs.rm(tempFilePath, { force: true }); -} diff --git a/desktop/src/main/services/ml-clip.ts b/desktop/src/main/services/ml-clip.ts index b4fa2c66d..622e880f8 100644 --- a/desktop/src/main/services/ml-clip.ts +++ b/desktop/src/main/services/ml-clip.ts @@ -11,8 +11,7 @@ import * as ort from "onnxruntime-node"; import Tokenizer from "../../thirdparty/clip-bpe-ts/mod"; import log from "../log"; import { writeStream } from "../stream"; -import { generateTempFilePath } from "../temp"; -import { deleteTempFile } from "./ffmpeg"; +import { deleteTempFile, generateTempFilePath } from "../utils-temp"; import { makeCachedInferenceSession } from "./ml"; const cachedCLIPImageSession = makeCachedInferenceSession( diff --git a/desktop/src/main/temp.ts b/desktop/src/main/temp.ts deleted file mode 100644 index 489e5cbd4..000000000 --- a/desktop/src/main/temp.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { app } from "electron/main"; -import { existsSync } from "node:fs"; -import fs from "node:fs/promises"; -import path from "path"; - -const CHARACTERS = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; - -export async function getTempDirPath() { - const tempDirPath = path.join(app.getPath("temp"), "ente"); - await fs.mkdir(tempDirPath, { recursive: true }); - return tempDirPath; -} - -function generateTempName(length: number) { - let result = ""; - - const charactersLength = CHARACTERS.length; - for (let i = 0; i < length; i++) { - result += CHARACTERS.charAt( - Math.floor(Math.random() * charactersLength), - ); - } - return result; -} - -export async function generateTempFilePath(formatSuffix: string) { - let tempFilePath: string; - do { - const tempDirPath = await getTempDirPath(); - const namePrefix = generateTempName(10); - tempFilePath = path.join(tempDirPath, namePrefix + "-" + formatSuffix); - } while (existsSync(tempFilePath)); - return tempFilePath; -} diff --git a/desktop/src/main/utils-temp.ts b/desktop/src/main/utils-temp.ts new file mode 100644 index 000000000..ddabc84d1 --- /dev/null +++ b/desktop/src/main/utils-temp.ts @@ -0,0 +1,64 @@ +import { app } from "electron/main"; +import { existsSync } from "node:fs"; +import fs from "node:fs/promises"; +import path from "path"; + +/** + * Our very own directory within the system temp directory. Go crazy, but + * remember to clean up, especially in exception handlers. + */ +const enteTempDirPath = async () => { + const result = path.join(app.getPath("temp"), "ente"); + await fs.mkdir(result, { recursive: true }); + return result; +}; + +const randomPrefix = (length: number) => { + const CHARACTERS = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + let result = ""; + const charactersLength = CHARACTERS.length; + for (let i = 0; i < length; i++) { + result += CHARACTERS.charAt( + Math.floor(Math.random() * charactersLength), + ); + } + return result; +}; + +/** + * Return the path to a temporary file with the given {@link formatSuffix}. + * + * The function returns the path to a file in the system temp directory (in an + * Ente specific folder therin) with a random prefix and the given + * {@link formatSuffix}. It ensures that there is no existing file with the same + * name already. + * + * Use {@link deleteTempFile} to remove this file when you're done. + */ +export const generateTempFilePath = async (formatSuffix: string) => { + const tempDir = await enteTempDirPath(); + let result: string; + do { + result = path.join(tempDir, randomPrefix(10) + "-" + formatSuffix); + } while (existsSync(result)); + return result; +}; + +/** + * Delete a temporary file at the given path if it exists. + * + * This is the same as a vanilla {@link fs.rm}, except it first checks that the + * given path is within the Ente specific directory in the system temp + * directory. This acts as an additional safety check. + * + * @param tempFilePath The path to the temporary file to delete. This path + * should've been previously created using {@link generateTempFilePath}. + */ +export const deleteTempFile = async (tempFilePath: string) => { + const tempDir = await enteTempDirPath(); + if (!tempFilePath.startsWith(tempDir)) + throw new Error(`Attempting to delete a non-temp file ${tempFilePath}`); + await fs.rm(tempFilePath, { force: true }); +}; diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 4c96295f5..5a7d0eda4 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -127,10 +127,10 @@ const fsIsDir = (dirPath: string): Promise => // - Conversion const convertToJPEG = ( - fileData: Uint8Array, - filename: string, + fileName: string, + imageData: Uint8Array, ): Promise => - ipcRenderer.invoke("convertToJPEG", fileData, filename); + ipcRenderer.invoke("convertToJPEG", fileName, imageData); const generateImageThumbnail = ( inputFile: File | ElectronFile, diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 9d0b83312..5d4bcf2e3 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,5 +1,4 @@ import { decodeLivePhoto } from "@/media/live-photo"; -import { convertBytesToHumanReadable } from "@/next/file"; import log from "@/next/log"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; import { workerBridge } from "@/next/worker/worker-bridge"; @@ -68,7 +67,7 @@ class ModuleState { * * Note the double negative when it is used. */ - isElectronJPEGConversionNotAvailable = false; + isNativeJPEGConversionNotAvailable = false; } const moduleState = new ModuleState(); @@ -282,7 +281,10 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { try { const tempFile = new File([imageBlob], fileName); fileTypeInfo = await getFileType(tempFile); - log.debug(() => `Obtaining renderable image for ${JSON.stringify(fileTypeInfo)}`); + log.debug( + () => + `Obtaining renderable image for ${JSON.stringify(fileTypeInfo)}`, + ); const { exactType } = fileTypeInfo; if (!isRawFile(exactType)) { @@ -292,18 +294,15 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { let jpegBlob: Blob | undefined; - const available = !moduleState.isElectronJPEGConversionNotAvailable; + const available = !moduleState.isNativeJPEGConversionNotAvailable; if (isElectron() && available && isSupportedRawFormat(exactType)) { // If we're running in our desktop app, see if our Node.js layer can // convert this into a JPEG using native tools for us. try { - jpegBlob = await tryConvertToJPEGInElectron( - imageBlob, - fileName, - ); + jpegBlob = await nativeConvertToJPEG(fileName, imageBlob); } catch (e) { if (e.message == CustomErrorMessage.NotAvailable) { - moduleState.isElectronJPEGConversionNotAvailable = true; + moduleState.isNativeJPEGConversionNotAvailable = true; } else { log.error("Native conversion to JPEG failed", e); throw e; @@ -326,27 +325,18 @@ export const getRenderableImage = async (fileName: string, imageBlob: Blob) => { } }; -const tryConvertToJPEGInElectron = async ( - fileBlob: Blob, - filename: string, -): Promise => { +const nativeConvertToJPEG = async (fileName: string, imageBlob: Blob) => { const startTime = Date.now(); - const inputFileData = new Uint8Array(await fileBlob.arrayBuffer()); + const imageData = new Uint8Array(await imageBlob.arrayBuffer()); const electron = globalThis.electron; // If we're running in a worker, we need to reroute the request back to // the main thread since workers don't have access to the `window` (and // thus, to the `window.electron`) object. - const convertedFileData = electron - ? await electron.convertToJPEG(inputFileData, filename) - : await workerBridge.convertToJPEG(inputFileData, filename); - log.info( - `originalFileSize:${convertBytesToHumanReadable( - fileBlob?.size, - )},convertedFileSize:${convertBytesToHumanReadable( - convertedFileData?.length, - )}, native conversion time: ${Date.now() - startTime}ms `, - ); - return new Blob([convertedFileData]); + const jpegData = electron + ? await electron.convertToJPEG(fileName, imageData) + : await workerBridge.convertToJPEG(fileName, imageData); + log.info(`Native JPEG conversion took ${Date.now() - startTime} ms`); + return new Blob([jpegData]); }; export function isFileHEIC(exactType: string) { diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 80ea174a5..8a2482ce0 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -191,26 +191,26 @@ export interface Electron { isDir: (dirPath: string) => Promise; }; - /* - * TODO: AUDIT below this - Some of the types we use below are not copyable - * across process boundaries, and such functions will (expectedly) fail at - * runtime. For such functions, find an efficient alternative or refactor - * the dataflow. - */ - // - Conversion /** - * Try to convert an arbitrary image into JPEG. + * Try to convert an arbitrary image into JPEG using native layer tools. * - * The behaviour is OS dependent. - * @param fileData - * @param filename - * @returns + * The behaviour is OS dependent. On macOS we use the `sips` utility, and on + * some Linux architectures we use an ImageMagick binary bundled with our + * desktop app. + * + * In other cases (primarily Windows), where native JPEG conversion is not + * yet possible, this method will throw an error with the + * {@link CustomErrorMessage.NotAvailable} message.. + * + * @param fileName The name of the file whose data we're being given. + * @param imageData The raw image data (the contents of the image file). + * @returns JPEG data. */ convertToJPEG: ( - fileData: Uint8Array, - filename: string, + fileName: string, + imageData: Uint8Array, ) => Promise; generateImageThumbnail: ( @@ -439,6 +439,13 @@ export interface Electron { filePaths: string[], ) => Promise; + /* + * TODO: AUDIT below this - Some of the types we use below are not copyable + * across process boundaries, and such functions will (expectedly) fail at + * runtime. For such functions, find an efficient alternative or refactor + * the dataflow. + */ + // - getElectronFilesFromGoogleZip: ( diff --git a/web/packages/next/worker/comlink-worker.ts b/web/packages/next/worker/comlink-worker.ts index a5237fccc..b687b9752 100644 --- a/web/packages/next/worker/comlink-worker.ts +++ b/web/packages/next/worker/comlink-worker.ts @@ -43,15 +43,16 @@ export class ComlinkWorker InstanceType> { * `workerBridge` object after importing it from `worker-bridge.ts`. * * Not all workers need access to all these functions, and this can indeed be - * done in a more fine-grained, per-worker, manner if needed. + * done in a more fine-grained, per-worker, manner if needed. For now, since it + * is a motley bunch, we just inject them all. */ const workerBridge = { // Needed: generally (presumably) logToDisk, // Needed by ML worker getAuthToken: () => ensureLocalUser().then((user) => user.token), - convertToJPEG: (inputFileData: Uint8Array, filename: string) => - ensureElectron().convertToJPEG(inputFileData, filename), + convertToJPEG: (fileName: string, imageData: Uint8Array) => + ensureElectron().convertToJPEG(fileName, imageData), detectFaces: (input: Float32Array) => ensureElectron().detectFaces(input), faceEmbedding: (input: Float32Array) => ensureElectron().faceEmbedding(input),