diff --git a/web/apps/photos/src/components/ExportInProgress.tsx b/web/apps/photos/src/components/ExportInProgress.tsx index ce2da895c..280ae52d4 100644 --- a/web/apps/photos/src/components/ExportInProgress.tsx +++ b/web/apps/photos/src/components/ExportInProgress.tsx @@ -10,9 +10,9 @@ import { LinearProgress, styled, } from "@mui/material"; -import { ExportStage } from "constants/export"; import { t } from "i18next"; import { Trans } from "react-i18next"; +import { ExportStage } from "services/export"; import { ExportProgress } from "types/export"; export const ComfySpan = styled("span")` diff --git a/web/apps/photos/src/components/ExportModal.tsx b/web/apps/photos/src/components/ExportModal.tsx index 877dee90f..159c872e4 100644 --- a/web/apps/photos/src/components/ExportModal.tsx +++ b/web/apps/photos/src/components/ExportModal.tsx @@ -14,12 +14,11 @@ import { Switch, Typography, } from "@mui/material"; -import { ExportStage } from "constants/export"; import { t } from "i18next"; import isElectron from "is-electron"; import { AppContext } from "pages/_app"; import { useContext, useEffect, useState } from "react"; -import exportService from "services/export"; +import exportService, { ExportStage } from "services/export"; import { ExportProgress, ExportSettings } from "types/export"; import { EnteFile } from "types/file"; import { getExportDirectoryDoesNotExistMessage } from "utils/ui"; diff --git a/web/apps/photos/src/constants/export.ts b/web/apps/photos/src/constants/export.ts deleted file mode 100644 index cd6c0c0ee..000000000 --- a/web/apps/photos/src/constants/export.ts +++ /dev/null @@ -1,14 +0,0 @@ -export const ENTE_METADATA_FOLDER = "metadata"; - -export const ENTE_TRASH_FOLDER = "Trash"; - -export enum ExportStage { - INIT = 0, - MIGRATION = 1, - STARTING = 2, - EXPORTING_FILES = 3, - TRASHING_DELETED_FILES = 4, - RENAMING_COLLECTION_FOLDERS = 5, - TRASHING_DELETED_COLLECTIONS = 6, - FINISHED = 7, -} diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 06961d6c9..c31256f13 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -52,7 +52,7 @@ import "photoswipe/dist/photoswipe.css"; import { createContext, useEffect, useRef, useState } from "react"; import LoadingBar from "react-top-loading-bar"; import DownloadManager from "services/download"; -import exportService from "services/export"; +import exportService, { resumeExportsIfNeeded } from "services/export"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import { getFamilyPortalRedirectURL, @@ -64,7 +64,6 @@ import { NotificationAttributes, SetNotificationAttributes, } from "types/Notification"; -import { isExportInProgress } from "utils/export"; import { getMLSearchConfig, updateMLSearchConfig, @@ -214,37 +213,10 @@ export default function App({ Component, pageProps }: AppProps) { return; } const initExport = async () => { - try { - log.info("init export"); - const token = getToken(); - if (!token) { - log.info( - "User not logged in, not starting export continuous sync job", - ); - return; - } - await DownloadManager.init(APPS.PHOTOS, { token }); - const exportSettings = exportService.getExportSettings(); - if ( - !(await exportService.exportFolderExists( - exportSettings?.folder, - )) - ) { - return; - } - const exportRecord = await exportService.getExportRecord( - exportSettings.folder, - ); - if (exportSettings.continuousExport) { - exportService.enableContinuousExport(); - } - if (isExportInProgress(exportRecord.stage)) { - log.info("export was in progress, resuming"); - exportService.scheduleExport(); - } - } catch (e) { - log.error("init export failed", e); - } + const token = getToken(); + if (!token) return; + await DownloadManager.init(APPS.PHOTOS, { token }); + await resumeExportsIfNeeded(); }; initExport(); try { diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index f2e90139a..419db1587 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -3,56 +3,42 @@ import log from "@/next/log"; import { CustomError } from "@ente/shared/error"; import { Events, eventBus } from "@ente/shared/events"; import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage"; +import { formatDateTimeShort } from "@ente/shared/time/format"; import { User } from "@ente/shared/user/types"; import { sleep } from "@ente/shared/utils"; import QueueProcessor, { CancellationStatus, RequestCanceller, } from "@ente/shared/utils/queueProcessor"; -import { ExportStage } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { Collection } from "types/collection"; import { + CollectionExportNames, ExportProgress, ExportRecord, ExportSettings, ExportUIUpdaters, + FileExportNames, } from "types/export"; import { EnteFile } from "types/file"; +import { Metadata } from "types/upload"; import { constructCollectionNameMap, getCollectionUserFacingName, getNonEmptyPersonalCollections, } from "utils/collection"; -import { - convertCollectionIDExportNameObjectToMap, - convertFileIDExportNameObjectToMap, - getCollectionExportPath, - getCollectionExportedFiles, - getCollectionIDFromFileUID, - getDeletedExportedCollections, - getDeletedExportedFiles, - getExportRecordFileUID, - getFileExportPath, - getFileMetadataExportPath, - getGoogleLikeMetadataFile, - getLivePhotoExportName, - getMetadataFileExportPath, - getMetadataFolderExportPath, - getRenamedExportedCollections, - getTrashedFileExportPath, - getUnExportedFiles, - getUniqueCollectionExportName, - getUniqueFileExportName, - isLivePhotoExportName, - parseLivePhotoExportName, -} from "utils/export"; import { generateStreamFromArrayBuffer, getPersonalFiles, getUpdatedEXIFFileForDownload, mergeMetadata, + splitFilenameAndExtension, } from "utils/file"; +import { + ENTE_TRASH_FOLDER, + getUniqueCollectionExportName, + getUniqueFileExportName, +} from "utils/native-fs"; import { getAllLocalCollections } from "../collectionService"; import downloadManager from "../download"; import { getAllLocalFiles } from "../fileService"; @@ -63,6 +49,19 @@ const EXPORT_RECORD_FILE_NAME = "export_status.json"; export const ENTE_EXPORT_DIRECTORY = "ente Photos"; +export const ENTE_METADATA_FOLDER = "metadata"; + +export enum ExportStage { + INIT = 0, + MIGRATION = 1, + STARTING = 2, + EXPORTING_FILES = 3, + TRASHING_DELETED_FILES = 4, + RENAMING_COLLECTION_FOLDERS = 5, + TRASHING_DELETED_COLLECTIONS = 6, + FINISHED = 7, +} + export const NULL_EXPORT_RECORD: ExportRecord = { version: 3, lastAttemptTimestamp: null, @@ -484,11 +483,7 @@ class ExportService { await this.verifyExportFolderExists(exportFolder); const oldCollectionExportName = collectionIDExportNameMap.get(collection.id); - const oldCollectionExportPath = getCollectionExportPath( - exportFolder, - oldCollectionExportName, - ); - + const oldCollectionExportPath = `${exportFolder}/${oldCollectionExportName}`; const newCollectionExportName = await getUniqueCollectionExportName( exportFolder, @@ -497,11 +492,7 @@ class ExportService { log.info( `renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}`, ); - const newCollectionExportPath = getCollectionExportPath( - exportFolder, - newCollectionExportName, - ); - + const newCollectionExportPath = `${exportFolder}/${newCollectionExportName}`; await this.addCollectionExportedRecord( exportFolder, collection.id, @@ -587,10 +578,7 @@ class ExportService { "collection is not empty, can't remove", ); } - const collectionExportPath = getCollectionExportPath( - exportFolder, - collectionExportName, - ); + const collectionExportPath = `${exportFolder}/${collectionExportName}`; await this.removeCollectionExportedRecord( exportFolder, collectionID, @@ -682,10 +670,7 @@ class ExportService { collectionExportName, ); } - const collectionExportPath = getCollectionExportPath( - exportDir, - collectionExportName, - ); + const collectionExportPath = `${exportDir}/${collectionExportName}`; await ensureElectron().checkExistsAndCreateDir( collectionExportPath, ); @@ -750,10 +735,10 @@ class ExportService { try { const fileExportName = fileIDExportNameMap.get(fileUID); const collectionID = getCollectionIDFromFileUID(fileUID); - const collectionExportPath = getCollectionExportPath( - exportDir, - collectionIDExportNameMap.get(collectionID), - ); + const collectionExportName = + collectionIDExportNameMap.get(collectionID); + const collectionExportPath = `${exportDir}/${collectionExportName}`; + await this.removeFileExportedRecord(exportDir, fileUID); try { if (isLivePhotoExportName(fileExportName)) { @@ -761,10 +746,7 @@ class ExportService { image: imageExportName, video: videoExportName, } = parseLivePhotoExportName(fileExportName); - const imageExportPath = getFileExportPath( - collectionExportPath, - imageExportName, - ); + const imageExportPath = `${collectionExportPath}/${imageExportName}`; log.info( `moving image file ${imageExportPath} to trash folder`, ); @@ -793,10 +775,7 @@ class ExportService { ); } - const videoExportPath = getFileExportPath( - collectionExportPath, - videoExportName, - ); + const videoExportPath = `${collectionExportPath}/${videoExportName}`; log.info( `moving video file ${videoExportPath} to trash folder`, ); @@ -823,10 +802,7 @@ class ExportService { ); } } else { - const fileExportPath = getFileExportPath( - collectionExportPath, - fileExportName, - ); + const fileExportPath = `${collectionExportPath}/${fileExportName}`; const trashedFilePath = await getTrashedFileExportPath( exportDir, @@ -1037,10 +1013,7 @@ class ExportService { exportFolder, collectionName, ); - const collectionExportPath = getCollectionExportPath( - exportFolder, - collectionExportName, - ); + const collectionExportPath = `${exportFolder}/${collectionExportName}`; await ensureElectron().checkExistsAndCreateDir(collectionExportPath); await ensureElectron().checkExistsAndCreateDir( getMetadataFolderExportPath(collectionExportPath), @@ -1090,7 +1063,7 @@ class ExportService { file, ); await ensureElectron().saveStreamToDisk( - getFileExportPath(collectionExportPath, fileExportName), + `${collectionExportPath}/${fileExportName}`, updatedFileStream, ); } catch (e) { @@ -1138,7 +1111,7 @@ class ExportService { file, ); await ensureElectron().saveStreamToDisk( - getFileExportPath(collectionExportPath, imageExportName), + `${collectionExportPath}/${imageExportName}`, imageStream, ); @@ -1150,12 +1123,12 @@ class ExportService { ); try { await ensureElectron().saveStreamToDisk( - getFileExportPath(collectionExportPath, videoExportName), + `${collectionExportPath}/${videoExportName}`, videoStream, ); } catch (e) { await ensureElectron().deleteFile( - getFileExportPath(collectionExportPath, imageExportName), + `${collectionExportPath}/${imageExportName}`, ); throw e; } @@ -1218,4 +1191,261 @@ class ExportService { return exportRecord; }; } -export default new ExportService(); + +const exportService = new ExportService(); + +export default exportService; + +/** + * If there are any in-progress exports, or if continuous exports are enabled, + * resume them. + */ +export const resumeExportsIfNeeded = async () => { + const exportSettings = exportService.getExportSettings(); + if (!(await exportService.exportFolderExists(exportSettings?.folder))) { + return; + } + const exportRecord = await exportService.getExportRecord( + exportSettings.folder, + ); + if (exportSettings.continuousExport) { + exportService.enableContinuousExport(); + } + if (isExportInProgress(exportRecord.stage)) { + log.debug(() => "Resuming in-progress export"); + exportService.scheduleExport(); + } +}; + +export const getExportRecordFileUID = (file: EnteFile) => + `${file.id}_${file.collectionID}_${file.updationTime}`; + +export const getCollectionIDFromFileUID = (fileUID: string) => + Number(fileUID.split("_")[1]); + +const convertCollectionIDExportNameObjectToMap = ( + collectionExportNames: CollectionExportNames, +): Map => { + return new Map( + Object.entries(collectionExportNames ?? {}).map((e) => { + return [Number(e[0]), String(e[1])]; + }), + ); +}; + +const convertFileIDExportNameObjectToMap = ( + fileExportNames: FileExportNames, +): Map => { + return new Map( + Object.entries(fileExportNames ?? {}).map((e) => { + return [String(e[0]), String(e[1])]; + }), + ); +}; + +const getRenamedExportedCollections = ( + collections: Collection[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.collectionExportNames) { + return []; + } + const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap( + exportRecord.collectionExportNames, + ); + const renamedCollections = collections.filter((collection) => { + if (collectionIDExportNameMap.has(collection.id)) { + const currentExportName = collectionIDExportNameMap.get( + collection.id, + ); + + const collectionExportName = + getCollectionUserFacingName(collection); + + if (currentExportName === collectionExportName) { + return false; + } + const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/); + const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix + ? currentExportName.replace(/\(\d+\)$/, "") + : currentExportName; + + return ( + collectionExportName !== currentExportNameWithoutNumberedSuffix + ); + } + return false; + }); + return renamedCollections; +}; + +const getDeletedExportedCollections = ( + collections: Collection[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.collectionExportNames) { + return []; + } + const presentCollections = new Set( + collections.map((collection) => collection.id), + ); + const deletedExportedCollections = Object.keys( + exportRecord?.collectionExportNames, + ) + .map(Number) + .filter((collectionID) => { + if (!presentCollections.has(collectionID)) { + return true; + } + return false; + }); + return deletedExportedCollections; +}; + +const getUnExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecord, +) => { + if (!exportRecord?.fileExportNames) { + return allFiles; + } + const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames)); + const unExportedFiles = allFiles.filter((file) => { + if (!exportedFiles.has(getExportRecordFileUID(file))) { + return true; + } + return false; + }); + return unExportedFiles; +}; + +const getDeletedExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecord, +): string[] => { + if (!exportRecord?.fileExportNames) { + return []; + } + const presentFileUIDs = new Set( + allFiles?.map((file) => getExportRecordFileUID(file)), + ); + const deletedExportedFiles = Object.keys( + exportRecord?.fileExportNames, + ).filter((fileUID) => { + if (!presentFileUIDs.has(fileUID)) { + return true; + } + return false; + }); + return deletedExportedFiles; +}; + +const getCollectionExportedFiles = ( + exportRecord: ExportRecord, + collectionID: number, +): string[] => { + if (!exportRecord?.fileExportNames) { + return []; + } + const collectionExportedFiles = Object.keys( + exportRecord?.fileExportNames, + ).filter((fileUID) => { + const fileCollectionID = Number(fileUID.split("_")[1]); + if (fileCollectionID === collectionID) { + return true; + } else { + return false; + } + }); + return collectionExportedFiles; +}; + +const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { + const metadata: Metadata = file.metadata; + const creationTime = Math.floor(metadata.creationTime / 1000000); + const modificationTime = Math.floor( + (metadata.modificationTime ?? metadata.creationTime) / 1000000, + ); + const captionValue: string = file?.pubMagicMetadata?.data?.caption; + return JSON.stringify( + { + title: fileExportName, + caption: captionValue, + creationTime: { + timestamp: creationTime, + formatted: formatDateTimeShort(creationTime * 1000), + }, + modificationTime: { + timestamp: modificationTime, + formatted: formatDateTimeShort(modificationTime * 1000), + }, + geoData: { + latitude: metadata.latitude, + longitude: metadata.longitude, + }, + }, + null, + 2, + ); +}; + +export const getMetadataFolderExportPath = (collectionExportPath: string) => + `${collectionExportPath}/${ENTE_METADATA_FOLDER}`; + +const getFileMetadataExportPath = ( + collectionExportPath: string, + fileExportName: string, +) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; + +const getTrashedFileExportPath = async (exportDir: string, path: string) => { + const fileRelativePath = path.replace(`${exportDir}/`, ""); + let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`; + let count = 1; + while (await exportService.exists(trashedFilePath)) { + const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath); + if (trashedFilePathParts[1]) { + trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`; + } else { + trashedFilePath = `${trashedFilePathParts[0]}(${count})`; + } + count++; + } + return trashedFilePath; +}; + +// if filepath is /home/user/Ente/Export/Collection1/1.jpg +// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json +const getMetadataFileExportPath = (filePath: string) => { + // extract filename and collection folder path + const filename = filePath.split("/").pop(); + const collectionExportPath = filePath.replace(`/${filename}`, ""); + return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`; +}; + +export const getLivePhotoExportName = ( + imageExportName: string, + videoExportName: string, +) => + JSON.stringify({ + image: imageExportName, + video: videoExportName, + }); + +export const isLivePhotoExportName = (exportName: string) => { + try { + JSON.parse(exportName); + return true; + } catch (e) { + return false; + } +}; + +const parseLivePhotoExportName = ( + livePhotoExportName: string, +): { image: string; video: string } => { + const { image, video } = JSON.parse(livePhotoExportName); + return { image, video }; +}; + +const isExportInProgress = (exportStage: ExportStage) => + exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED; diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index 6c79420ed..49265cf34 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -15,34 +15,25 @@ import { ExportRecordV0, ExportRecordV1, ExportRecordV2, + ExportedCollectionPaths, FileExportNames, } from "types/export"; import { EnteFile } from "types/file"; import { getNonEmptyPersonalCollections } from "utils/collection"; -import { - getCollectionExportPath, - getCollectionIDFromFileUID, - getExportRecordFileUID, - getLivePhotoExportName, - getMetadataFolderExportPath, -} from "utils/export"; -import { - convertCollectionIDFolderPathObjectToMap, - getExportedFiles, - getFileMetadataSavePath, - getFileSavePath, - getOldCollectionFolderPath, - getOldFileMetadataSavePath, - getOldFileSavePath, - getUniqueCollectionFolderPath, - getUniqueFileExportNameForMigration, - getUniqueFileSaveName, -} from "utils/export/migration"; +import { splitFilenameAndExtension } from "utils/ffmpeg"; import { getIDBasedSortedFiles, getPersonalFiles, mergeMetadata, } from "utils/file"; +import { sanitizeName } from "utils/native-fs"; +import { + ENTE_METADATA_FOLDER, + getCollectionIDFromFileUID, + getExportRecordFileUID, + getLivePhotoExportName, + getMetadataFolderExportPath, +} from "."; import exportService from "./index"; export async function migrateExport( @@ -441,7 +432,7 @@ async function removeCollectionExportMissingMetadataFolder( if ( await exportService.exists( getMetadataFolderExportPath( - getCollectionExportPath(exportDir, collectionExportName), + `${exportDir}/${collectionExportName}`, ), ) ) { @@ -475,3 +466,130 @@ async function removeCollectionExportMissingMetadataFolder( }; await exportService.updateExportRecord(exportDir, updatedExportRecord); } + +const convertCollectionIDFolderPathObjectToMap = ( + exportedCollectionPaths: ExportedCollectionPaths, +): Map => { + return new Map( + Object.entries(exportedCollectionPaths ?? {}).map((e) => { + return [Number(e[0]), String(e[1])]; + }), + ); +}; + +const getExportedFiles = ( + allFiles: EnteFile[], + exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2, +) => { + if (!exportRecord?.exportedFiles) { + return []; + } + const exportedFileIds = new Set(exportRecord?.exportedFiles); + const exportedFiles = allFiles.filter((file) => { + if (exportedFileIds.has(getExportRecordFileUID(file))) { + return true; + } else { + return false; + } + }); + return exportedFiles; +}; + +const oldSanitizeName = (name: string) => + name.replaceAll("/", "_").replaceAll(" ", "_"); + +const getUniqueCollectionFolderPath = async ( + dir: string, + collectionName: string, +): Promise => { + let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; + let count = 1; + while (await exportService.exists(collectionFolderPath)) { + collectionFolderPath = `${dir}/${sanitizeName( + collectionName, + )}(${count})`; + count++; + } + return collectionFolderPath; +}; + +export const getMetadataFolderPath = (collectionFolderPath: string) => + `${collectionFolderPath}/${ENTE_METADATA_FOLDER}`; + +const getUniqueFileSaveName = async ( + collectionPath: string, + filename: string, +) => { + let fileSaveName = sanitizeName(filename); + let count = 1; + while ( + await exportService.exists( + getFileSavePath(collectionPath, fileSaveName), + ) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileSaveName = `${filenameParts[0]}(${count})`; + } + count++; + } + return fileSaveName; +}; + +const getFileMetadataSavePath = ( + collectionFolderPath: string, + fileSaveName: string, +) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`; + +const getFileSavePath = (collectionFolderPath: string, fileSaveName: string) => + `${collectionFolderPath}/${fileSaveName}`; + +const getOldCollectionFolderPath = ( + dir: string, + collectionID: number, + collectionName: string, +) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`; + +const getOldFileSavePath = (collectionFolderPath: string, file: EnteFile) => + `${collectionFolderPath}/${file.id}_${oldSanitizeName( + file.metadata.title, + )}`; + +const getOldFileMetadataSavePath = ( + collectionFolderPath: string, + file: EnteFile, +) => + `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${ + file.id + }_${oldSanitizeName(file.metadata.title)}.json`; + +const getUniqueFileExportNameForMigration = ( + collectionPath: string, + filename: string, + usedFilePaths: Map>, +) => { + let fileExportName = sanitizeName(filename); + let count = 1; + while ( + usedFilePaths + .get(collectionPath) + ?.has(getFileSavePath(collectionPath, fileExportName)) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileExportName = `${filenameParts[0]}(${count})`; + } + count++; + } + if (!usedFilePaths.has(collectionPath)) { + usedFilePaths.set(collectionPath, new Set()); + } + usedFilePaths + .get(collectionPath) + .add(getFileSavePath(collectionPath, fileExportName)); + return fileExportName; +}; diff --git a/web/apps/photos/src/types/export/index.ts b/web/apps/photos/src/types/export/index.ts index ce85f32fd..64ef249ed 100644 --- a/web/apps/photos/src/types/export/index.ts +++ b/web/apps/photos/src/types/export/index.ts @@ -1,4 +1,4 @@ -import { ExportStage } from "constants/export"; +import type { ExportStage } from "services/export"; import { EnteFile } from "types/file"; export interface ExportProgress { diff --git a/web/apps/photos/src/utils/collection/index.ts b/web/apps/photos/src/utils/collection/index.ts index c18861515..581523828 100644 --- a/web/apps/photos/src/utils/collection/index.ts +++ b/web/apps/photos/src/utils/collection/index.ts @@ -42,12 +42,9 @@ import { import { EnteFile } from "types/file"; import { SetFilesDownloadProgressAttributes } from "types/gallery"; import { SUB_TYPE, VISIBILITY_STATE } from "types/magicMetadata"; -import { - getCollectionExportPath, - getUniqueCollectionExportName, -} from "utils/export"; import { downloadFilesWithProgress } from "utils/file"; import { isArchivedCollection, updateMagicMetadata } from "utils/magicMetadata"; +import { getUniqueCollectionExportName } from "utils/native-fs"; export enum COLLECTION_OPS_TYPE { ADD, @@ -176,10 +173,7 @@ async function createCollectionDownloadFolder( downloadDirPath, collectionName, ); - const collectionDownloadPath = getCollectionExportPath( - downloadDirPath, - collectionDownloadName, - ); + const collectionDownloadPath = `${downloadDirPath}/${collectionDownloadName}`; await exportService.checkExistsAndCreateDir(collectionDownloadPath); return collectionDownloadPath; } diff --git a/web/apps/photos/src/utils/export/index.ts b/web/apps/photos/src/utils/export/index.ts deleted file mode 100644 index a98e431b2..000000000 --- a/web/apps/photos/src/utils/export/index.ts +++ /dev/null @@ -1,312 +0,0 @@ -import exportService from "services/export"; -import { Collection } from "types/collection"; -import { - CollectionExportNames, - ExportRecord, - FileExportNames, -} from "types/export"; - -import { EnteFile } from "types/file"; - -import { formatDateTimeShort } from "@ente/shared/time/format"; -import { - ENTE_METADATA_FOLDER, - ENTE_TRASH_FOLDER, - ExportStage, -} from "constants/export"; -import sanitize from "sanitize-filename"; -import { Metadata } from "types/upload"; -import { getCollectionUserFacingName } from "utils/collection"; -import { splitFilenameAndExtension } from "utils/file"; - -export const getExportRecordFileUID = (file: EnteFile) => - `${file.id}_${file.collectionID}_${file.updationTime}`; - -export const getCollectionIDFromFileUID = (fileUID: string) => - Number(fileUID.split("_")[1]); - -export const convertCollectionIDExportNameObjectToMap = ( - collectionExportNames: CollectionExportNames, -): Map => { - return new Map( - Object.entries(collectionExportNames ?? {}).map((e) => { - return [Number(e[0]), String(e[1])]; - }), - ); -}; - -export const convertFileIDExportNameObjectToMap = ( - fileExportNames: FileExportNames, -): Map => { - return new Map( - Object.entries(fileExportNames ?? {}).map((e) => { - return [String(e[0]), String(e[1])]; - }), - ); -}; - -export const getRenamedExportedCollections = ( - collections: Collection[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.collectionExportNames) { - return []; - } - const collectionIDExportNameMap = convertCollectionIDExportNameObjectToMap( - exportRecord.collectionExportNames, - ); - const renamedCollections = collections.filter((collection) => { - if (collectionIDExportNameMap.has(collection.id)) { - const currentExportName = collectionIDExportNameMap.get( - collection.id, - ); - - const collectionExportName = - getCollectionUserFacingName(collection); - - if (currentExportName === collectionExportName) { - return false; - } - const hasNumberedSuffix = currentExportName.match(/\(\d+\)$/); - const currentExportNameWithoutNumberedSuffix = hasNumberedSuffix - ? currentExportName.replace(/\(\d+\)$/, "") - : currentExportName; - - return ( - collectionExportName !== currentExportNameWithoutNumberedSuffix - ); - } - return false; - }); - return renamedCollections; -}; - -export const getDeletedExportedCollections = ( - collections: Collection[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.collectionExportNames) { - return []; - } - const presentCollections = new Set( - collections.map((collection) => collection.id), - ); - const deletedExportedCollections = Object.keys( - exportRecord?.collectionExportNames, - ) - .map(Number) - .filter((collectionID) => { - if (!presentCollections.has(collectionID)) { - return true; - } - return false; - }); - return deletedExportedCollections; -}; - -export const getUnExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecord, -) => { - if (!exportRecord?.fileExportNames) { - return allFiles; - } - const exportedFiles = new Set(Object.keys(exportRecord?.fileExportNames)); - const unExportedFiles = allFiles.filter((file) => { - if (!exportedFiles.has(getExportRecordFileUID(file))) { - return true; - } - return false; - }); - return unExportedFiles; -}; - -export const getDeletedExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecord, -): string[] => { - if (!exportRecord?.fileExportNames) { - return []; - } - const presentFileUIDs = new Set( - allFiles?.map((file) => getExportRecordFileUID(file)), - ); - const deletedExportedFiles = Object.keys( - exportRecord?.fileExportNames, - ).filter((fileUID) => { - if (!presentFileUIDs.has(fileUID)) { - return true; - } - return false; - }); - return deletedExportedFiles; -}; - -export const getCollectionExportedFiles = ( - exportRecord: ExportRecord, - collectionID: number, -): string[] => { - if (!exportRecord?.fileExportNames) { - return []; - } - const collectionExportedFiles = Object.keys( - exportRecord?.fileExportNames, - ).filter((fileUID) => { - const fileCollectionID = Number(fileUID.split("_")[1]); - if (fileCollectionID === collectionID) { - return true; - } else { - return false; - } - }); - return collectionExportedFiles; -}; - -export const getGoogleLikeMetadataFile = ( - fileExportName: string, - file: EnteFile, -) => { - const metadata: Metadata = file.metadata; - const creationTime = Math.floor(metadata.creationTime / 1000000); - const modificationTime = Math.floor( - (metadata.modificationTime ?? metadata.creationTime) / 1000000, - ); - const captionValue: string = file?.pubMagicMetadata?.data?.caption; - return JSON.stringify( - { - title: fileExportName, - caption: captionValue, - creationTime: { - timestamp: creationTime, - formatted: formatDateTimeShort(creationTime * 1000), - }, - modificationTime: { - timestamp: modificationTime, - formatted: formatDateTimeShort(modificationTime * 1000), - }, - geoData: { - latitude: metadata.latitude, - longitude: metadata.longitude, - }, - }, - null, - 2, - ); -}; - -export const sanitizeName = (name: string) => - sanitize(name, { replacement: "_" }); - -export const getUniqueCollectionExportName = async ( - dir: string, - collectionName: string, -): Promise => { - let collectionExportName = sanitizeName(collectionName); - let count = 1; - while ( - (await exportService.exists( - getCollectionExportPath(dir, collectionExportName), - )) || - collectionExportName === ENTE_TRASH_FOLDER - ) { - collectionExportName = `${sanitizeName(collectionName)}(${count})`; - count++; - } - return collectionExportName; -}; - -export const getMetadataFolderExportPath = (collectionExportPath: string) => - `${collectionExportPath}/${ENTE_METADATA_FOLDER}`; - -export const getUniqueFileExportName = async ( - collectionExportPath: string, - filename: string, -) => { - let fileExportName = sanitizeName(filename); - let count = 1; - while ( - await exportService.exists( - getFileExportPath(collectionExportPath, fileExportName), - ) - ) { - const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); - if (filenameParts[1]) { - fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; - } else { - fileExportName = `${filenameParts[0]}(${count})`; - } - count++; - } - return fileExportName; -}; - -export const getFileMetadataExportPath = ( - collectionExportPath: string, - fileExportName: string, -) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`; - -export const getCollectionExportPath = ( - exportFolder: string, - collectionExportName: string, -) => `${exportFolder}/${collectionExportName}`; - -export const getFileExportPath = ( - collectionExportPath: string, - fileExportName: string, -) => `${collectionExportPath}/${fileExportName}`; - -export const getTrashedFileExportPath = async ( - exportDir: string, - path: string, -) => { - const fileRelativePath = path.replace(`${exportDir}/`, ""); - let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`; - let count = 1; - while (await exportService.exists(trashedFilePath)) { - const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath); - if (trashedFilePathParts[1]) { - trashedFilePath = `${trashedFilePathParts[0]}(${count}).${trashedFilePathParts[1]}`; - } else { - trashedFilePath = `${trashedFilePathParts[0]}(${count})`; - } - count++; - } - return trashedFilePath; -}; - -// if filepath is /home/user/Ente/Export/Collection1/1.jpg -// then metadata path is /home/user/Ente/Export/Collection1/ENTE_METADATA_FOLDER/1.jpg.json -export const getMetadataFileExportPath = (filePath: string) => { - // extract filename and collection folder path - const filename = filePath.split("/").pop(); - const collectionExportPath = filePath.replace(`/${filename}`, ""); - return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`; -}; - -export const getLivePhotoExportName = ( - imageExportName: string, - videoExportName: string, -) => - JSON.stringify({ - image: imageExportName, - video: videoExportName, - }); - -export const isLivePhotoExportName = (exportName: string) => { - try { - JSON.parse(exportName); - return true; - } catch (e) { - return false; - } -}; - -export const parseLivePhotoExportName = ( - livePhotoExportName: string, -): { image: string; video: string } => { - const { image, video } = JSON.parse(livePhotoExportName); - return { image, video }; -}; - -export const isExportInProgress = (exportStage: ExportStage) => - exportStage > ExportStage.INIT && exportStage < ExportStage.FINISHED; diff --git a/web/apps/photos/src/utils/export/migration.ts b/web/apps/photos/src/utils/export/migration.ts deleted file mode 100644 index c8988cac4..000000000 --- a/web/apps/photos/src/utils/export/migration.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { ENTE_METADATA_FOLDER } from "constants/export"; -import exportService from "services/export"; -import { - ExportedCollectionPaths, - ExportRecordV0, - ExportRecordV1, - ExportRecordV2, -} from "types/export"; -import { EnteFile } from "types/file"; -import { splitFilenameAndExtension } from "utils/ffmpeg"; -import { getExportRecordFileUID, sanitizeName } from "."; - -export const convertCollectionIDFolderPathObjectToMap = ( - exportedCollectionPaths: ExportedCollectionPaths, -): Map => { - return new Map( - Object.entries(exportedCollectionPaths ?? {}).map((e) => { - return [Number(e[0]), String(e[1])]; - }), - ); -}; - -export const getExportedFiles = ( - allFiles: EnteFile[], - exportRecord: ExportRecordV0 | ExportRecordV1 | ExportRecordV2, -) => { - if (!exportRecord?.exportedFiles) { - return []; - } - const exportedFileIds = new Set(exportRecord?.exportedFiles); - const exportedFiles = allFiles.filter((file) => { - if (exportedFileIds.has(getExportRecordFileUID(file))) { - return true; - } else { - return false; - } - }); - return exportedFiles; -}; - -export const oldSanitizeName = (name: string) => - name.replaceAll("/", "_").replaceAll(" ", "_"); - -export const getUniqueCollectionFolderPath = async ( - dir: string, - collectionName: string, -): Promise => { - let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`; - let count = 1; - while (await exportService.exists(collectionFolderPath)) { - collectionFolderPath = `${dir}/${sanitizeName( - collectionName, - )}(${count})`; - count++; - } - return collectionFolderPath; -}; - -export const getMetadataFolderPath = (collectionFolderPath: string) => - `${collectionFolderPath}/${ENTE_METADATA_FOLDER}`; - -export const getUniqueFileSaveName = async ( - collectionPath: string, - filename: string, -) => { - let fileSaveName = sanitizeName(filename); - let count = 1; - while ( - await exportService.exists( - getFileSavePath(collectionPath, fileSaveName), - ) - ) { - const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); - if (filenameParts[1]) { - fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; - } else { - fileSaveName = `${filenameParts[0]}(${count})`; - } - count++; - } - return fileSaveName; -}; - -export const getOldFileSaveName = (filename: string, fileID: number) => - `${fileID}_${oldSanitizeName(filename)}`; - -export const getFileMetadataSavePath = ( - collectionFolderPath: string, - fileSaveName: string, -) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`; - -export const getFileSavePath = ( - collectionFolderPath: string, - fileSaveName: string, -) => `${collectionFolderPath}/${fileSaveName}`; - -export const getOldCollectionFolderPath = ( - dir: string, - collectionID: number, - collectionName: string, -) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`; - -export const getOldFileSavePath = ( - collectionFolderPath: string, - file: EnteFile, -) => - `${collectionFolderPath}/${file.id}_${oldSanitizeName( - file.metadata.title, - )}`; - -export const getOldFileMetadataSavePath = ( - collectionFolderPath: string, - file: EnteFile, -) => - `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${ - file.id - }_${oldSanitizeName(file.metadata.title)}.json`; - -export const getUniqueFileExportNameForMigration = ( - collectionPath: string, - filename: string, - usedFilePaths: Map>, -) => { - let fileExportName = sanitizeName(filename); - let count = 1; - while ( - usedFilePaths - .get(collectionPath) - ?.has(getFileSavePath(collectionPath, fileExportName)) - ) { - const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); - if (filenameParts[1]) { - fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; - } else { - fileExportName = `${filenameParts[0]}(${count})`; - } - count++; - } - if (!usedFilePaths.has(collectionPath)) { - usedFilePaths.set(collectionPath, new Set()); - } - usedFilePaths - .get(collectionPath) - .add(getFileSavePath(collectionPath, fileExportName)); - return fileExportName; -}; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 089c5f40d..cd432ecbe 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -51,8 +51,8 @@ import { } from "types/gallery"; import { VISIBILITY_STATE } from "types/magicMetadata"; import { FileTypeInfo } from "types/upload"; -import { getFileExportPath, getUniqueFileExportName } from "utils/export"; import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata"; +import { getUniqueFileExportName } from "utils/native-fs"; const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000; @@ -818,7 +818,7 @@ async function downloadFileDesktop( ); const imageStream = generateStreamFromArrayBuffer(livePhoto.image); await electron.saveStreamToDisk( - getFileExportPath(downloadPath, imageExportName), + `${downloadPath}/${imageExportName}`, imageStream, ); try { @@ -828,13 +828,11 @@ async function downloadFileDesktop( ); const videoStream = generateStreamFromArrayBuffer(livePhoto.video); await electron.saveStreamToDisk( - getFileExportPath(downloadPath, videoExportName), + `${downloadPath}/${videoExportName}`, videoStream, ); } catch (e) { - await electron.deleteFile( - getFileExportPath(downloadPath, imageExportName), - ); + await electron.deleteFile(`${downloadPath}/${imageExportName}`); throw e; } } else { @@ -843,7 +841,7 @@ async function downloadFileDesktop( file.metadata.title, ); await electron.saveStreamToDisk( - getFileExportPath(downloadPath, fileExportName), + `${downloadPath}/${fileExportName}`, updatedFileStream, ); } diff --git a/web/apps/photos/src/utils/native-fs.ts b/web/apps/photos/src/utils/native-fs.ts new file mode 100644 index 000000000..4173aa7ac --- /dev/null +++ b/web/apps/photos/src/utils/native-fs.ts @@ -0,0 +1,44 @@ +import sanitize from "sanitize-filename"; +import exportService from "services/export"; +import { splitFilenameAndExtension } from "utils/file"; + +export const ENTE_TRASH_FOLDER = "Trash"; + +export const sanitizeName = (name: string) => + sanitize(name, { replacement: "_" }); + +export const getUniqueCollectionExportName = async ( + dir: string, + collectionName: string, +): Promise => { + let collectionExportName = sanitizeName(collectionName); + let count = 1; + while ( + (await exportService.exists(`${dir}/${collectionExportName}`)) || + collectionExportName === ENTE_TRASH_FOLDER + ) { + collectionExportName = `${sanitizeName(collectionName)}(${count})`; + count++; + } + return collectionExportName; +}; + +export const getUniqueFileExportName = async ( + collectionExportPath: string, + filename: string, +) => { + let fileExportName = sanitizeName(filename); + let count = 1; + while ( + await exportService.exists(`${collectionExportPath}/${fileExportName}`) + ) { + const filenameParts = splitFilenameAndExtension(sanitizeName(filename)); + if (filenameParts[1]) { + fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`; + } else { + fileExportName = `${filenameParts[0]}(${count})`; + } + count++; + } + return fileExportName; +}; diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 6cce03aa9..708ec5dcf 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -1,4 +1,3 @@ -import { ENTE_METADATA_FOLDER } from "constants/export"; import { FILE_TYPE } from "constants/file"; import { A_SEC_IN_MICROSECONDS, @@ -6,6 +5,7 @@ import { PICKED_UPLOAD_TYPE, } from "constants/upload"; import isElectron from "is-electron"; +import { ENTE_METADATA_FOLDER } from "services/export"; import { EnteFile } from "types/file"; import { ElectronFile,