[desktop] Fix export related IPC - Part 1/x (#1432)

web/apps/photos/src/components/ExportInProgress.tsx

@@ -10,9 +10,9 @@ import {
     LinearProgress,
     styled,
 } from "@mui/material";
-import { ExportStage } from "constants/export";
 import { t } from "i18next";
 import { Trans } from "react-i18next";
+import { ExportStage } from "services/export";
 import { ExportProgress } from "types/export";
 
 export const ComfySpan = styled("span")`

web/apps/photos/src/components/ExportModal.tsx

@@ -14,12 +14,11 @@ import {
     Switch,
     Typography,
 } from "@mui/material";
-import { ExportStage } from "constants/export";
 import { t } from "i18next";
 import isElectron from "is-electron";
 import { AppContext } from "pages/_app";
 import { useContext, useEffect, useState } from "react";
-import exportService from "services/export";
+import exportService, { ExportStage } from "services/export";
 import { ExportProgress, ExportSettings } from "types/export";
 import { EnteFile } from "types/file";
 import { getExportDirectoryDoesNotExistMessage } from "utils/ui";

web/apps/photos/src/constants/export.ts

@@ -1,14 +0,0 @@
-export const ENTE_METADATA_FOLDER = "metadata";
-
-export const ENTE_TRASH_FOLDER = "Trash";
-
-export enum ExportStage {
-    INIT = 0,
-    MIGRATION = 1,
-    STARTING = 2,
-    EXPORTING_FILES = 3,
-    TRASHING_DELETED_FILES = 4,
-    RENAMING_COLLECTION_FOLDERS = 5,
-    TRASHING_DELETED_COLLECTIONS = 6,
-    FINISHED = 7,
-}

web/apps/photos/src/pages/_app.tsx

@@ -52,7 +52,7 @@ import "photoswipe/dist/photoswipe.css";
 import { createContext, useEffect, useRef, useState } from "react";
 import LoadingBar from "react-top-loading-bar";
 import DownloadManager from "services/download";
-import exportService from "services/export";
+import exportService, { resumeExportsIfNeeded } from "services/export";
 import mlWorkManager from "services/machineLearning/mlWorkManager";
 import {
     getFamilyPortalRedirectURL,
@@ -64,7 +64,6 @@ import {
     NotificationAttributes,
     SetNotificationAttributes,
 } from "types/Notification";
-import { isExportInProgress } from "utils/export";
 import {
     getMLSearchConfig,
     updateMLSearchConfig,
@@ -214,37 +213,10 @@ export default function App({ Component, pageProps }: AppProps) {
             return;
         }
         const initExport = async () => {
-            try {
-                log.info("init export");
-                const token = getToken();
-                if (!token) {
-                    log.info(
-                        "User not logged in, not starting export continuous sync job",
-                    );
-                    return;
-                }
-                await DownloadManager.init(APPS.PHOTOS, { token });
-                const exportSettings = exportService.getExportSettings();
-                if (
-                    !(await exportService.exportFolderExists(
-                        exportSettings?.folder,
-                    ))
-                ) {
-                    return;
-                }
-                const exportRecord = await exportService.getExportRecord(
-                    exportSettings.folder,
-                );
-                if (exportSettings.continuousExport) {
-                    exportService.enableContinuousExport();
-                }
-                if (isExportInProgress(exportRecord.stage)) {
-                    log.info("export was in progress, resuming");
-                    exportService.scheduleExport();
-                }
-            } catch (e) {
-                log.error("init export failed", e);
-            }
+            const token = getToken();
+            if (!token) return;
+            await DownloadManager.init(APPS.PHOTOS, { token });
+            await resumeExportsIfNeeded();
         };
         initExport();
         try {

web/apps/photos/src/services/export/index.ts

@@ -3,56 +3,42 @@ import log from "@/next/log";
 import { CustomError } from "@ente/shared/error";
 import { Events, eventBus } from "@ente/shared/events";
 import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage";
+import { formatDateTimeShort } from "@ente/shared/time/format";
 import { User } from "@ente/shared/user/types";
 import { sleep } from "@ente/shared/utils";
 import QueueProcessor, {
     CancellationStatus,
     RequestCanceller,
 } from "@ente/shared/utils/queueProcessor";
-import { ExportStage } from "constants/export";
 import { FILE_TYPE } from "constants/file";
 import { Collection } from "types/collection";
 import {
+    CollectionExportNames,
     ExportProgress,
     ExportRecord,
     ExportSettings,
     ExportUIUpdaters,
+    FileExportNames,
 } from "types/export";
 import { EnteFile } from "types/file";
+import { Metadata } from "types/upload";
 import {
     constructCollectionNameMap,
     getCollectionUserFacingName,
     getNonEmptyPersonalCollections,
 } from "utils/collection";
-import {
-    convertCollectionIDExportNameObjectToMap,
-    convertFileIDExportNameObjectToMap,
-    getCollectionExportPath,
-    getCollectionExportedFiles,
-    getCollectionIDFromFileUID,
-    getDeletedExportedCollections,
-    getDeletedExportedFiles,
-    getExportRecordFileUID,
-    getFileExportPath,
-    getFileMetadataExportPath,
-    getGoogleLikeMetadataFile,
-    getLivePhotoExportName,
-    getMetadataFileExportPath,
-    getMetadataFolderExportPath,
-    getRenamedExportedCollections,
-    getTrashedFileExportPath,
-    getUnExportedFiles,
-    getUniqueCollectionExportName,
-    getUniqueFileExportName,
-    isLivePhotoExportName,
-    parseLivePhotoExportName,
-} from "utils/export";
 import {
     generateStreamFromArrayBuffer,
     getPersonalFiles,
     getUpdatedEXIFFileForDownload,
     mergeMetadata,
+    splitFilenameAndExtension,
 } from "utils/file";
+import {
+    ENTE_TRASH_FOLDER,
+    getUniqueCollectionExportName,
+    getUniqueFileExportName,
+} from "utils/native-fs";
 import { getAllLocalCollections } from "../collectionService";
 import downloadManager from "../download";
 import { getAllLocalFiles } from "../fileService";
@@ -63,6 +49,19 @@ const EXPORT_RECORD_FILE_NAME = "export_status.json";
 
 export const ENTE_EXPORT_DIRECTORY = "ente Photos";
 
+export const ENTE_METADATA_FOLDER = "metadata";
+
+export enum ExportStage {
+    INIT = 0,
+    MIGRATION = 1,
+    STARTING = 2,
+    EXPORTING_FILES = 3,
+    TRASHING_DELETED_FILES = 4,
+    RENAMING_COLLECTION_FOLDERS = 5,
+    TRASHING_DELETED_COLLECTIONS = 6,
+    FINISHED = 7,
+}
+
 export const NULL_EXPORT_RECORD: ExportRecord = {
     version: 3,
     lastAttemptTimestamp: null,
@@ -484,11 +483,7 @@ class ExportService {
                     await this.verifyExportFolderExists(exportFolder);
                     const oldCollectionExportName =
                         collectionIDExportNameMap.get(collection.id);
-                    const oldCollectionExportPath = getCollectionExportPath(
-                        exportFolder,
-                        oldCollectionExportName,
-                    );
-
+                    const oldCollectionExportPath = `${exportFolder}/${oldCollectionExportName}`;
                     const newCollectionExportName =
                         await getUniqueCollectionExportName(
                             exportFolder,
@@ -497,11 +492,7 @@ class ExportService {
                     log.info(
                         `renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}`,
                     );
-                    const newCollectionExportPath = getCollectionExportPath(
-                        exportFolder,
-                        newCollectionExportName,
-                    );
-
+                    const newCollectionExportPath = `${exportFolder}/${newCollectionExportName}`;
                     await this.addCollectionExportedRecord(
                         exportFolder,
                         collection.id,
@@ -587,10 +578,7 @@ class ExportService {
                             "collection is not empty, can't remove",
                         );
                     }
-                    const collectionExportPath = getCollectionExportPath(
-                        exportFolder,
-                        collectionExportName,
-                    );
+                    const collectionExportPath = `${exportFolder}/${collectionExportName}`;
                     await this.removeCollectionExportedRecord(
                         exportFolder,
                         collectionID,
@@ -682,10 +670,7 @@ class ExportService {
                             collectionExportName,
                         );
                     }
-                    const collectionExportPath = getCollectionExportPath(
-                        exportDir,
-                        collectionExportName,
-                    );
+                    const collectionExportPath = `${exportDir}/${collectionExportName}`;
                     await ensureElectron().checkExistsAndCreateDir(
                         collectionExportPath,
                     );
@@ -750,10 +735,10 @@ class ExportService {
                 try {
                     const fileExportName = fileIDExportNameMap.get(fileUID);
                     const collectionID = getCollectionIDFromFileUID(fileUID);
-                    const collectionExportPath = getCollectionExportPath(
-                        exportDir,
-                        collectionIDExportNameMap.get(collectionID),
-                    );
+                    const collectionExportName =
+                        collectionIDExportNameMap.get(collectionID);
+                    const collectionExportPath = `${exportDir}/${collectionExportName}`;
+
                     await this.removeFileExportedRecord(exportDir, fileUID);
                     try {
                         if (isLivePhotoExportName(fileExportName)) {
@@ -761,10 +746,7 @@ class ExportService {
                                 image: imageExportName,
                                 video: videoExportName,
                             } = parseLivePhotoExportName(fileExportName);
-                            const imageExportPath = getFileExportPath(
-                                collectionExportPath,
-                                imageExportName,
-                            );
+                            const imageExportPath = `${collectionExportPath}/${imageExportName}`;
                             log.info(
                                 `moving image file ${imageExportPath} to trash folder`,
                             );
@@ -793,10 +775,7 @@ class ExportService {
                                 );
                             }
 
-                            const videoExportPath = getFileExportPath(
-                                collectionExportPath,
-                                videoExportName,
-                            );
+                            const videoExportPath = `${collectionExportPath}/${videoExportName}`;
                             log.info(
                                 `moving video file ${videoExportPath} to trash folder`,
                             );
@@ -823,10 +802,7 @@ class ExportService {
                                 );
                             }
                         } else {
-                            const fileExportPath = getFileExportPath(
-                                collectionExportPath,
-                                fileExportName,
-                            );
+                            const fileExportPath = `${collectionExportPath}/${fileExportName}`;
                             const trashedFilePath =
                                 await getTrashedFileExportPath(
                                     exportDir,
@@ -1037,10 +1013,7 @@ class ExportService {
             exportFolder,
             collectionName,
         );
-        const collectionExportPath = getCollectionExportPath(
-            exportFolder,
-            collectionExportName,
-        );
+        const collectionExportPath = `${exportFolder}/${collectionExportName}`;
         await ensureElectron().checkExistsAndCreateDir(collectionExportPath);
         await ensureElectron().checkExistsAndCreateDir(
             getMetadataFolderExportPath(collectionExportPath),
@@ -1090,7 +1063,7 @@ class ExportService {
                         file,
                     );
                     await ensureElectron().saveStreamToDisk(
-                        getFileExportPath(collectionExportPath, fileExportName),
+                        `${collectionExportPath}/${fileExportName}`,
                         updatedFileStream,
                     );
                 } catch (e) {
@@ -1138,7 +1111,7 @@ class ExportService {
                 file,
             );
             await ensureElectron().saveStreamToDisk(
-                getFileExportPath(collectionExportPath, imageExportName),
+                `${collectionExportPath}/${imageExportName}`,
                 imageStream,
             );
 
@@ -1150,12 +1123,12 @@ class ExportService {
             );
             try {
                 await ensureElectron().saveStreamToDisk(
-                    getFileExportPath(collectionExportPath, videoExportName),
+                    `${collectionExportPath}/${videoExportName}`,
                     videoStream,
                 );
             } catch (e) {
                 await ensureElectron().deleteFile(
-                    getFileExportPath(collectionExportPath, imageExportName),
+                    `${collectionExportPath}/${imageExportName}`,
                 );
                 throw e;
             }
@@ -1218,4 +1191,261 @@ class ExportService {
         return exportRecord;
     };
 }
-export default new ExportService();
+
+const exportService = new ExportService();
+
+export default exportService;
+
+/**
+ * If there are any in-progress exports, or if continuous exports are enabled,
+ * resume them.
+ */
+export const resumeExportsIfNeeded = async () => {
+    const exportSettings = exportService.getExportSettings();
+    if (!(await exportService.exportFolderExists(exportSettings?.folder))) {
+        return;
+    }
+    const exportRecord = await exportService.getExportRecord(
+        exportSettings.folder,
+    );
+    if (exportSettings.continuousExport) {
+        exportService.enableContinuousExport();
+    }
+    if (isExportInProgress(exportRecord.stage)) {
+        log.debug(() => "Resuming in-progress export");
+        exportService.scheduleExport();
+    }
+};
+
+export const getExportRecordFileUID = (file: EnteFile) =>
+    `${file.id}_${file.collectionID}_${file.updationTime}`;
+
+export const getCollectionIDFromFileUID = (fileUID: string) =>
+    Number(fileUID.split("_")[1]);
+
+const convertCollectionIDExportNameObjectToMap = (
+    collectionExportNames: CollectionExportNames,
+): Map<number, string> => {
+    return new Map<number, string>(
+        Object.entries(collectionExportNames ?? {}).map((e) => {
+            return [Number(e[0]), String(e[1])];
+        }),
+    );
+};
+
+const convertFileIDExportNameObjectToMap = (
+    fileExportNames: FileExportNames,
+): Map<string, string> => {
+    return new Map<string, string>(
+        Object.entries(fileExportNames ?? {}).map((e) => {
+            return [String(e[0]), String(e[1])];
+        }),
+    );
+};
+
+const getRenamedExportedCollections = (
+    collections: Collection[],
+    exportRecord: ExportRecord,
+) => {
+    if (!exportRecord?.collectionExportNames) {
+        return [];
+    }
+    const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap(
+        exportRecord.collectionExportNames,
+    );
+    const renamedCollections = collections.filter((collection) => {
+        if (collectionIDExportNameMap.has(collection.id)) {
+            const currentExportName = collectionIDExportNameMap.get(
+                collection.id,
+            );
+
+            const collectionExportName =
+                getCollectionUserFacingName(collection);
+
+            if (currentExportName === collectionExportName) {
+                return false;
+            }
+            const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/);
+            const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix
+                ? currentExportName.replace(/\(\d+\)$/, "")
+                : currentExportName;
+
+            return (
+                collectionExportName !== currentExportNameWithoutNumberedSuffix
+            );
+        }
+        return false;
+    });
+    return renamedCollections;
+};
+
+const getDeletedExportedCollections = (
+    collections: Collection[],
+    exportRecord: ExportRecord,
+) => {
+    if (!exportRecord?.collectionExportNames) {
+        return [];
+    }
+    const presentCollections = new Set(
+        collections.map((collection) => collection.id),
+    );
+    const deletedExportedCollections = Object.keys(
+        exportRecord?.collectionExportNames,
+    )
+        .map(Number)
+        .filter((collectionID) => {
+            if (!presentCollections.has(collectionID)) {
+                return true;
+            }
+            return false;
+        });
+    return deletedExportedCollections;
+};
+
+const getUnExportedFiles = (
+    allFiles: EnteFile[],
+    exportRecord: ExportRecord,
+) => {
+    if (!exportRecord?.fileExportNames) {
+        return allFiles;
+    }
+    const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames));
+    const unExportedFiles = allFiles.filter((file) => {
+        if (!exportedFiles.has(getExportRecordFileUID(file))) {
+            return true;
+        }
+        return false;
+    });
+    return unExportedFiles;
+};
+
+const getDeletedExportedFiles = (
+    allFiles: EnteFile[],
+    exportRecord: ExportRecord,
+): string[] => {
+    if (!exportRecord?.fileExportNames) {
+        return [];
+    }
+    const presentFileUIDs = new Set(
+        allFiles?.map((file) => getExportRecordFileUID(file)),
+    );
+    const deletedExportedFiles = Object.keys(
+        exportRecord?.fileExportNames,
+    ).filter((fileUID) => {
+        if (!presentFileUIDs.has(fileUID)) {
+            return true;
+        }
+        return false;
+    });
+    return deletedExportedFiles;
+};
+
+const getCollectionExportedFiles = (
+    exportRecord: ExportRecord,
+    collectionID: number,
+): string[] => {
+    if (!exportRecord?.fileExportNames) {
+        return [];
+    }
+    const collectionExportedFiles = Object.keys(
+        exportRecord?.fileExportNames,
+    ).filter((fileUID) => {
+        const fileCollectionID = Number(fileUID.split("_")[1]);
+        if (fileCollectionID === collectionID) {
+            return true;
+        } else {
+            return false;
+        }
+    });
+    return collectionExportedFiles;
+};
+
+const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => {
+    const metadata: Metadata = file.metadata;
+    const creationTime = Math.floor(metadata.creationTime / 1000000);
+    const modificationTime = Math.floor(
+        (metadata.modificationTime ?? metadata.creationTime) / 1000000,
+    );
+    const captionValue: string = file?.pubMagicMetadata?.data?.caption;
+    return JSON.stringify(
+        {
+            title: fileExportName,
+            caption: captionValue,
+            creationTime: {
+                timestamp: creationTime,
+                formatted: formatDateTimeShort(creationTime * 1000),
+            },
+            modificationTime: {
+                timestamp: modificationTime,
+                formatted: formatDateTimeShort(modificationTime * 1000),
+            },
+            geoData: {
+                latitude: metadata.latitude,
+                longitude: metadata.longitude,
+            },
+        },
+        null,
+        2,
+    );
+};
+
+export const getMetadataFolderExportPath = (collectionExportPath: string) =>
+    `${collectionExportPath}/${ENTE_METADATA_FOLDER}`;
+
+const getFileMetadataExportPath = (
+    collectionExportPath: string,
+    fileExportName: string,
+) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`;
+
+const getTrashedFileExportPath = async (exportDir: string, path: string) => {
+    const fileRelativePath = path.replace(`${exportDir}/`, "");
+    let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`;
+    let count = 1;
+    while (await exportService.exists(trashedFilePath)) {
+        const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath);
+        if (trashedFilePathParts[1]) {
+            trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`;
+        } else {
+            trashedFilePath = `${trashedFilePathParts[0]}(${count})`;
+        }
+        count++;
+    }
+    return trashedFilePath;
+};
+
+// if filepath is /home/user/Ente/Export/Collection1/1.jpg
+// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json
+const getMetadataFileExportPath = (filePath: string) => {
+    // extract filename and collection folder path
+    const filename = filePath.split("/").pop();
+    const collectionExportPath = filePath.replace(`/${filename}`, "");
+    return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`;
+};
+
+export const getLivePhotoExportName = (
+    imageExportName: string,
+    videoExportName: string,
+) =>
+    JSON.stringify({
+        image: imageExportName,
+        video: videoExportName,
+    });
+
+export const isLivePhotoExportName = (exportName: string) => {
+    try {
+        JSON.parse(exportName);
+        return true;
+    } catch (e) {
+        return false;
+    }
+};
+
+const parseLivePhotoExportName = (
+    livePhotoExportName: string,
+): { image: string; video: string } => {
+    const { image, video } = JSON.parse(livePhotoExportName);
+    return { image, video };
+};
+
+const isExportInProgress = (exportStage: ExportStage) =>
+    exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED;

web/apps/photos/src/services/export/migration.ts

@@ -15,34 +15,25 @@ import {
     ExportRecordV0,
     ExportRecordV1,
     ExportRecordV2,
+    ExportedCollectionPaths,
     FileExportNames,
 } from "types/export";
 import { EnteFile } from "types/file";
 import { getNonEmptyPersonalCollections } from "utils/collection";
-import {
-    getCollectionExportPath,
-    getCollectionIDFromFileUID,
-    getExportRecordFileUID,
-    getLivePhotoExportName,
-    getMetadataFolderExportPath,
-} from "utils/export";
-import {
-    convertCollectionIDFolderPathObjectToMap,
-    getExportedFiles,
-    getFileMetadataSavePath,
-    getFileSavePath,
-    getOldCollectionFolderPath,
-    getOldFileMetadataSavePath,
-    getOldFileSavePath,
-    getUniqueCollectionFolderPath,
-    getUniqueFileExportNameForMigration,
-    getUniqueFileSaveName,
-} from "utils/export/migration";
+import { splitFilenameAndExtension } from "utils/ffmpeg";
 import {
     getIDBasedSortedFiles,
     getPersonalFiles,
     mergeMetadata,
 } from "utils/file";
+import { sanitizeName } from "utils/native-fs";
+import {
+    ENTE_METADATA_FOLDER,
+    getCollectionIDFromFileUID,
+    getExportRecordFileUID,
+    getLivePhotoExportName,
+    getMetadataFolderExportPath,
+} from ".";
 import exportService from "./index";
 
 export async function migrateExport(
@@ -441,7 +432,7 @@ async function removeCollectionExportMissingMetadataFolder(
         if (
             await exportService.exists(
                 getMetadataFolderExportPath(
-                    getCollectionExportPath(exportDir, collectionExportName),
+                    `${exportDir}/${collectionExportName}`,
                 ),
             )
         ) {
@@ -475,3 +466,130 @@ async function removeCollectionExportMissingMetadataFolder(
     };
     await exportService.updateExportRecord(exportDir, updatedExportRecord);
 }
+
+const convertCollectionIDFolderPathObjectToMap = (
+    exportedCollectionPaths: ExportedCollectionPaths,
+): Map<number, string> => {
+    return new Map<number, string>(
+        Object.entries(exportedCollectionPaths ?? {}).map((e) => {
+            return [Number(e[0]), String(e[1])];
+        }),
+    );
+};
+
+const getExportedFiles = (
+    allFiles: EnteFile[],
+    exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2,
+) => {
+    if (!exportRecord?.exportedFiles) {
+        return [];
+    }
+    const exportedFileIds = new Set(exportRecord?.exportedFiles);
+    const exportedFiles = allFiles.filter((file) => {
+        if (exportedFileIds.has(getExportRecordFileUID(file))) {
+            return true;
+        } else {
+            return false;
+        }
+    });
+    return exportedFiles;
+};
+
+const oldSanitizeName = (name: string) =>
+    name.replaceAll("/", "_").replaceAll(" ", "_");
+
+const getUniqueCollectionFolderPath = async (
+    dir: string,
+    collectionName: string,
+): Promise<string> => {
+    let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
+    let count = 1;
+    while (await exportService.exists(collectionFolderPath)) {
+        collectionFolderPath = `${dir}/${sanitizeName(
+            collectionName,
+        )}(${count})`;
+        count++;
+    }
+    return collectionFolderPath;
+};
+
+export const getMetadataFolderPath = (collectionFolderPath: string) =>
+    `${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
+
+const getUniqueFileSaveName = async (
+    collectionPath: string,
+    filename: string,
+) => {
+    let fileSaveName = sanitizeName(filename);
+    let count = 1;
+    while (
+        await exportService.exists(
+            getFileSavePath(collectionPath, fileSaveName),
+        )
+    ) {
+        const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
+        if (filenameParts[1]) {
+            fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
+        } else {
+            fileSaveName = `${filenameParts[0]}(${count})`;
+        }
+        count++;
+    }
+    return fileSaveName;
+};
+
+const getFileMetadataSavePath = (
+    collectionFolderPath: string,
+    fileSaveName: string,
+) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
+
+const getFileSavePath = (collectionFolderPath: string, fileSaveName: string) =>
+    `${collectionFolderPath}/${fileSaveName}`;
+
+const getOldCollectionFolderPath = (
+    dir: string,
+    collectionID: number,
+    collectionName: string,
+) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
+
+const getOldFileSavePath = (collectionFolderPath: string, file: EnteFile) =>
+    `${collectionFolderPath}/${file.id}_${oldSanitizeName(
+        file.metadata.title,
+    )}`;
+
+const getOldFileMetadataSavePath = (
+    collectionFolderPath: string,
+    file: EnteFile,
+) =>
+    `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
+        file.id
+    }_${oldSanitizeName(file.metadata.title)}.json`;
+
+const getUniqueFileExportNameForMigration = (
+    collectionPath: string,
+    filename: string,
+    usedFilePaths: Map<string, Set<string>>,
+) => {
+    let fileExportName = sanitizeName(filename);
+    let count = 1;
+    while (
+        usedFilePaths
+            .get(collectionPath)
+            ?.has(getFileSavePath(collectionPath, fileExportName))
+    ) {
+        const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
+        if (filenameParts[1]) {
+            fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
+        } else {
+            fileExportName = `${filenameParts[0]}(${count})`;
+        }
+        count++;
+    }
+    if (!usedFilePaths.has(collectionPath)) {
+        usedFilePaths.set(collectionPath, new Set());
+    }
+    usedFilePaths
+        .get(collectionPath)
+        .add(getFileSavePath(collectionPath, fileExportName));
+    return fileExportName;
+};

web/apps/photos/src/types/export/index.ts

@@ -1,4 +1,4 @@
-import { ExportStage } from "constants/export";
+import type { ExportStage } from "services/export";
 import { EnteFile } from "types/file";
 
 export interface ExportProgress {

web/apps/photos/src/utils/collection/index.ts

@@ -42,12 +42,9 @@ import {
 import { EnteFile } from "types/file";
 import { SetFilesDownloadProgressAttributes } from "types/gallery";
 import { SUB_TYPE, VISIBILITY_STATE } from "types/magicMetadata";
-import {
-    getCollectionExportPath,
-    getUniqueCollectionExportName,
-} from "utils/export";
 import { downloadFilesWithProgress } from "utils/file";
 import { isArchivedCollection, updateMagicMetadata } from "utils/magicMetadata";
+import { getUniqueCollectionExportName } from "utils/native-fs";
 
 export enum COLLECTION_OPS_TYPE {
     ADD,
@@ -176,10 +173,7 @@ async function createCollectionDownloadFolder(
         downloadDirPath,
         collectionName,
     );
-    const collectionDownloadPath = getCollectionExportPath(
-        downloadDirPath,
-        collectionDownloadName,
-    );
+    const collectionDownloadPath = `${downloadDirPath}/${collectionDownloadName}`;
     await exportService.checkExistsAndCreateDir(collectionDownloadPath);
     return collectionDownloadPath;
 }

web/apps/photos/src/utils/export/index.ts

@@ -1,312 +0,0 @@
-import exportService from "services/export";
-import { Collection } from "types/collection";
-import {
-    CollectionExportNames,
-    ExportRecord,
-    FileExportNames,
-} from "types/export";
-
-import { EnteFile } from "types/file";
-
-import { formatDateTimeShort } from "@ente/shared/time/format";
-import {
-    ENTE_METADATA_FOLDER,
-    ENTE_TRASH_FOLDER,
-    ExportStage,
-} from "constants/export";
-import sanitize from "sanitize-filename";
-import { Metadata } from "types/upload";
-import { getCollectionUserFacingName } from "utils/collection";
-import { splitFilenameAndExtension } from "utils/file";
-
-export const getExportRecordFileUID = (file: EnteFile) =>
-    `${file.id}_${file.collectionID}_${file.updationTime}`;
-
-export const getCollectionIDFromFileUID = (fileUID: string) =>
-    Number(fileUID.split("_")[1]);
-
-export const convertCollectionIDExportNameObjectToMap = (
-    collectionExportNames: CollectionExportNames,
-): Map<number, string> => {
-    return new Map<number, string>(
-        Object.entries(collectionExportNames ?? {}).map((e) => {
-            return [Number(e[0]), String(e[1])];
-        }),
-    );
-};
-
-export const convertFileIDExportNameObjectToMap = (
-    fileExportNames: FileExportNames,
-): Map<string, string> => {
-    return new Map<string, string>(
-        Object.entries(fileExportNames ?? {}).map((e) => {
-            return [String(e[0]), String(e[1])];
-        }),
-    );
-};
-
-export const getRenamedExportedCollections = (
-    collections: Collection[],
-    exportRecord: ExportRecord,
-) => {
-    if (!exportRecord?.collectionExportNames) {
-        return [];
-    }
-    const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap(
-        exportRecord.collectionExportNames,
-    );
-    const renamedCollections = collections.filter((collection) => {
-        if (collectionIDExportNameMap.has(collection.id)) {
-            const currentExportName = collectionIDExportNameMap.get(
-                collection.id,
-            );
-
-            const collectionExportName =
-                getCollectionUserFacingName(collection);
-
-            if (currentExportName === collectionExportName) {
-                return false;
-            }
-            const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/);
-            const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix
-                ? currentExportName.replace(/\(\d+\)$/, "")
-                : currentExportName;
-
-            return (
-                collectionExportName !== currentExportNameWithoutNumberedSuffix
-            );
-        }
-        return false;
-    });
-    return renamedCollections;
-};
-
-export const getDeletedExportedCollections = (
-    collections: Collection[],
-    exportRecord: ExportRecord,
-) => {
-    if (!exportRecord?.collectionExportNames) {
-        return [];
-    }
-    const presentCollections = new Set(
-        collections.map((collection) => collection.id),
-    );
-    const deletedExportedCollections = Object.keys(
-        exportRecord?.collectionExportNames,
-    )
-        .map(Number)
-        .filter((collectionID) => {
-            if (!presentCollections.has(collectionID)) {
-                return true;
-            }
-            return false;
-        });
-    return deletedExportedCollections;
-};
-
-export const getUnExportedFiles = (
-    allFiles: EnteFile[],
-    exportRecord: ExportRecord,
-) => {
-    if (!exportRecord?.fileExportNames) {
-        return allFiles;
-    }
-    const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames));
-    const unExportedFiles = allFiles.filter((file) => {
-        if (!exportedFiles.has(getExportRecordFileUID(file))) {
-            return true;
-        }
-        return false;
-    });
-    return unExportedFiles;
-};
-
-export const getDeletedExportedFiles = (
-    allFiles: EnteFile[],
-    exportRecord: ExportRecord,
-): string[] => {
-    if (!exportRecord?.fileExportNames) {
-        return [];
-    }
-    const presentFileUIDs = new Set(
-        allFiles?.map((file) => getExportRecordFileUID(file)),
-    );
-    const deletedExportedFiles = Object.keys(
-        exportRecord?.fileExportNames,
-    ).filter((fileUID) => {
-        if (!presentFileUIDs.has(fileUID)) {
-            return true;
-        }
-        return false;
-    });
-    return deletedExportedFiles;
-};
-
-export const getCollectionExportedFiles = (
-    exportRecord: ExportRecord,
-    collectionID: number,
-): string[] => {
-    if (!exportRecord?.fileExportNames) {
-        return [];
-    }
-    const collectionExportedFiles = Object.keys(
-        exportRecord?.fileExportNames,
-    ).filter((fileUID) => {
-        const fileCollectionID = Number(fileUID.split("_")[1]);
-        if (fileCollectionID === collectionID) {
-            return true;
-        } else {
-            return false;
-        }
-    });
-    return collectionExportedFiles;
-};
-
-export const getGoogleLikeMetadataFile = (
-    fileExportName: string,
-    file: EnteFile,
-) => {
-    const metadata: Metadata = file.metadata;
-    const creationTime = Math.floor(metadata.creationTime / 1000000);
-    const modificationTime = Math.floor(
-        (metadata.modificationTime ?? metadata.creationTime) / 1000000,
-    );
-    const captionValue: string = file?.pubMagicMetadata?.data?.caption;
-    return JSON.stringify(
-        {
-            title: fileExportName,
-            caption: captionValue,
-            creationTime: {
-                timestamp: creationTime,
-                formatted: formatDateTimeShort(creationTime * 1000),
-            },
-            modificationTime: {
-                timestamp: modificationTime,
-                formatted: formatDateTimeShort(modificationTime * 1000),
-            },
-            geoData: {
-                latitude: metadata.latitude,
-                longitude: metadata.longitude,
-            },
-        },
-        null,
-        2,
-    );
-};
-
-export const sanitizeName = (name: string) =>
-    sanitize(name, { replacement: "_" });
-
-export const getUniqueCollectionExportName = async (
-    dir: string,
-    collectionName: string,
-): Promise<string> => {
-    let collectionExportName = sanitizeName(collectionName);
-    let count = 1;
-    while (
-        (await exportService.exists(
-            getCollectionExportPath(dir, collectionExportName),
-        )) ||
-        collectionExportName === ENTE_TRASH_FOLDER
-    ) {
-        collectionExportName = `${sanitizeName(collectionName)}(${count})`;
-        count++;
-    }
-    return collectionExportName;
-};
-
-export const getMetadataFolderExportPath = (collectionExportPath: string) =>
-    `${collectionExportPath}/${ENTE_METADATA_FOLDER}`;
-
-export const getUniqueFileExportName = async (
-    collectionExportPath: string,
-    filename: string,
-) => {
-    let fileExportName = sanitizeName(filename);
-    let count = 1;
-    while (
-        await exportService.exists(
-            getFileExportPath(collectionExportPath, fileExportName),
-        )
-    ) {
-        const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
-        if (filenameParts[1]) {
-            fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
-        } else {
-            fileExportName = `${filenameParts[0]}(${count})`;
-        }
-        count++;
-    }
-    return fileExportName;
-};
-
-export const getFileMetadataExportPath = (
-    collectionExportPath: string,
-    fileExportName: string,
-) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`;
-
-export const getCollectionExportPath = (
-    exportFolder: string,
-    collectionExportName: string,
-) => `${exportFolder}/${collectionExportName}`;
-
-export const getFileExportPath = (
-    collectionExportPath: string,
-    fileExportName: string,
-) => `${collectionExportPath}/${fileExportName}`;
-
-export const getTrashedFileExportPath = async (
-    exportDir: string,
-    path: string,
-) => {
-    const fileRelativePath = path.replace(`${exportDir}/`, "");
-    let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`;
-    let count = 1;
-    while (await exportService.exists(trashedFilePath)) {
-        const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath);
-        if (trashedFilePathParts[1]) {
-            trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`;
-        } else {
-            trashedFilePath = `${trashedFilePathParts[0]}(${count})`;
-        }
-        count++;
-    }
-    return trashedFilePath;
-};
-
-// if filepath is /home/user/Ente/Export/Collection1/1.jpg
-// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json
-export const getMetadataFileExportPath = (filePath: string) => {
-    // extract filename and collection folder path
-    const filename = filePath.split("/").pop();
-    const collectionExportPath = filePath.replace(`/${filename}`, "");
-    return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`;
-};
-
-export const getLivePhotoExportName = (
-    imageExportName: string,
-    videoExportName: string,
-) =>
-    JSON.stringify({
-        image: imageExportName,
-        video: videoExportName,
-    });
-
-export const isLivePhotoExportName = (exportName: string) => {
-    try {
-        JSON.parse(exportName);
-        return true;
-    } catch (e) {
-        return false;
-    }
-};
-
-export const parseLivePhotoExportName = (
-    livePhotoExportName: string,
-): { image: string; video: string } => {
-    const { image, video } = JSON.parse(livePhotoExportName);
-    return { image, video };
-};
-
-export const isExportInProgress = (exportStage: ExportStage) =>
-    exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED;

web/apps/photos/src/utils/export/migration.ts

@@ -1,146 +0,0 @@
-import { ENTE_METADATA_FOLDER } from "constants/export";
-import exportService from "services/export";
-import {
-    ExportedCollectionPaths,
-    ExportRecordV0,
-    ExportRecordV1,
-    ExportRecordV2,
-} from "types/export";
-import { EnteFile } from "types/file";
-import { splitFilenameAndExtension } from "utils/ffmpeg";
-import { getExportRecordFileUID, sanitizeName } from ".";
-
-export const convertCollectionIDFolderPathObjectToMap = (
-    exportedCollectionPaths: ExportedCollectionPaths,
-): Map<number, string> => {
-    return new Map<number, string>(
-        Object.entries(exportedCollectionPaths ?? {}).map((e) => {
-            return [Number(e[0]), String(e[1])];
-        }),
-    );
-};
-
-export const getExportedFiles = (
-    allFiles: EnteFile[],
-    exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2,
-) => {
-    if (!exportRecord?.exportedFiles) {
-        return [];
-    }
-    const exportedFileIds = new Set(exportRecord?.exportedFiles);
-    const exportedFiles = allFiles.filter((file) => {
-        if (exportedFileIds.has(getExportRecordFileUID(file))) {
-            return true;
-        } else {
-            return false;
-        }
-    });
-    return exportedFiles;
-};
-
-export const oldSanitizeName = (name: string) =>
-    name.replaceAll("/", "_").replaceAll(" ", "_");
-
-export const getUniqueCollectionFolderPath = async (
-    dir: string,
-    collectionName: string,
-): Promise<string> => {
-    let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
-    let count = 1;
-    while (await exportService.exists(collectionFolderPath)) {
-        collectionFolderPath = `${dir}/${sanitizeName(
-            collectionName,
-        )}(${count})`;
-        count++;
-    }
-    return collectionFolderPath;
-};
-
-export const getMetadataFolderPath = (collectionFolderPath: string) =>
-    `${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
-
-export const getUniqueFileSaveName = async (
-    collectionPath: string,
-    filename: string,
-) => {
-    let fileSaveName = sanitizeName(filename);
-    let count = 1;
-    while (
-        await exportService.exists(
-            getFileSavePath(collectionPath, fileSaveName),
-        )
-    ) {
-        const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
-        if (filenameParts[1]) {
-            fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
-        } else {
-            fileSaveName = `${filenameParts[0]}(${count})`;
-        }
-        count++;
-    }
-    return fileSaveName;
-};
-
-export const getOldFileSaveName = (filename: string, fileID: number) =>
-    `${fileID}_${oldSanitizeName(filename)}`;
-
-export const getFileMetadataSavePath = (
-    collectionFolderPath: string,
-    fileSaveName: string,
-) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
-
-export const getFileSavePath = (
-    collectionFolderPath: string,
-    fileSaveName: string,
-) => `${collectionFolderPath}/${fileSaveName}`;
-
-export const getOldCollectionFolderPath = (
-    dir: string,
-    collectionID: number,
-    collectionName: string,
-) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
-
-export const getOldFileSavePath = (
-    collectionFolderPath: string,
-    file: EnteFile,
-) =>
-    `${collectionFolderPath}/${file.id}_${oldSanitizeName(
-        file.metadata.title,
-    )}`;
-
-export const getOldFileMetadataSavePath = (
-    collectionFolderPath: string,
-    file: EnteFile,
-) =>
-    `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
-        file.id
-    }_${oldSanitizeName(file.metadata.title)}.json`;
-
-export const getUniqueFileExportNameForMigration = (
-    collectionPath: string,
-    filename: string,
-    usedFilePaths: Map<string, Set<string>>,
-) => {
-    let fileExportName = sanitizeName(filename);
-    let count = 1;
-    while (
-        usedFilePaths
-            .get(collectionPath)
-            ?.has(getFileSavePath(collectionPath, fileExportName))
-    ) {
-        const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
-        if (filenameParts[1]) {
-            fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
-        } else {
-            fileExportName = `${filenameParts[0]}(${count})`;
-        }
-        count++;
-    }
-    if (!usedFilePaths.has(collectionPath)) {
-        usedFilePaths.set(collectionPath, new Set());
-    }
-    usedFilePaths
-        .get(collectionPath)
-        .add(getFileSavePath(collectionPath, fileExportName));
-    return fileExportName;
-};

web/apps/photos/src/utils/file/index.ts

@@ -51,8 +51,8 @@ import {
 } from "types/gallery";
 import { VISIBILITY_STATE } from "types/magicMetadata";
 import { FileTypeInfo } from "types/upload";
-import { getFileExportPath, getUniqueFileExportName } from "utils/export";
 import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata";
+import { getUniqueFileExportName } from "utils/native-fs";
 
 const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000;
 
@@ -818,7 +818,7 @@ async function downloadFileDesktop(
         );
         const imageStream = generateStreamFromArrayBuffer(livePhoto.image);
         await electron.saveStreamToDisk(
-            getFileExportPath(downloadPath, imageExportName),
+            `${downloadPath}/${imageExportName}`,
             imageStream,
         );
         try {
@@ -828,13 +828,11 @@ async function downloadFileDesktop(
             );
             const videoStream = generateStreamFromArrayBuffer(livePhoto.video);
             await electron.saveStreamToDisk(
-                getFileExportPath(downloadPath, videoExportName),
+                `${downloadPath}/${videoExportName}`,
                 videoStream,
             );
         } catch (e) {
-            await electron.deleteFile(
-                getFileExportPath(downloadPath, imageExportName),
-            );
+            await electron.deleteFile(`${downloadPath}/${imageExportName}`);
             throw e;
         }
     } else {
@@ -843,7 +841,7 @@ async function downloadFileDesktop(
             file.metadata.title,
         );
         await electron.saveStreamToDisk(
-            getFileExportPath(downloadPath, fileExportName),
+            `${downloadPath}/${fileExportName}`,
             updatedFileStream,
         );
     }

web/apps/photos/src/utils/native-fs.ts

@@ -0,0 +1,44 @@
+import sanitize from "sanitize-filename";
+import exportService from "services/export";
+import { splitFilenameAndExtension } from "utils/file";
+
+export const ENTE_TRASH_FOLDER = "Trash";
+
+export const sanitizeName = (name: string) =>
+    sanitize(name, { replacement: "_" });
+
+export const getUniqueCollectionExportName = async (
+    dir: string,
+    collectionName: string,
+): Promise<string> => {
+    let collectionExportName = sanitizeName(collectionName);
+    let count = 1;
+    while (
+        (await exportService.exists(`${dir}/${collectionExportName}`)) ||
+        collectionExportName === ENTE_TRASH_FOLDER
+    ) {
+        collectionExportName = `${sanitizeName(collectionName)}(${count})`;
+        count++;
+    }
+    return collectionExportName;
+};
+
+export const getUniqueFileExportName = async (
+    collectionExportPath: string,
+    filename: string,
+) => {
+    let fileExportName = sanitizeName(filename);
+    let count = 1;
+    while (
+        await exportService.exists(`${collectionExportPath}/${fileExportName}`)
+    ) {
+        const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
+        if (filenameParts[1]) {
+            fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
+        } else {
+            fileExportName = `${filenameParts[0]}(${count})`;
+        }
+        count++;
+    }
+    return fileExportName;
+};

web/apps/photos/src/utils/upload/index.ts

@@ -1,4 +1,3 @@
-import { ENTE_METADATA_FOLDER } from "constants/export";
 import { FILE_TYPE } from "constants/file";
 import {
     A_SEC_IN_MICROSECONDS,
@@ -6,6 +5,7 @@ import {
     PICKED_UPLOAD_TYPE,
 } from "constants/upload";
 import isElectron from "is-electron";
+import { ENTE_METADATA_FOLDER } from "services/export";
 import { EnteFile } from "types/file";
 import {
     ElectronFile,