diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index abbe54705..df6ab7c8e 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -53,9 +53,9 @@ import { } from "./services/store"; import { clearPendingUploads, - listZipEntries, + listZipItems, markUploadedFiles, - markUploadedZipEntries, + markUploadedZipItems, pathOrZipItemSize, pendingUploads, setPendingUploads, @@ -205,8 +205,8 @@ export const attachIPCHandlers = () => { // - Upload - ipcMain.handle("listZipEntries", (_, zipPath: string) => - listZipEntries(zipPath), + ipcMain.handle("listZipItems", (_, zipPath: string) => + listZipItems(zipPath), ); ipcMain.handle("pathOrZipItemSize", (_, pathOrZipItem: string | ZipItem) => @@ -225,9 +225,8 @@ export const attachIPCHandlers = () => { ); ipcMain.handle( - "markUploadedZipEntries", - (_, zipEntries: PendingUploads["zipItems"]) => - markUploadedZipEntries(zipEntries), + "markUploadedZipItems", + (_, items: PendingUploads["zipItems"]) => markUploadedZipItems(items), ); ipcMain.handle("clearPendingUploads", () => clearPendingUploads()); diff --git a/desktop/src/main/services/upload.ts b/desktop/src/main/services/upload.ts index 1a1d343c5..9b24cc0ea 100644 --- a/desktop/src/main/services/upload.ts +++ b/desktop/src/main/services/upload.ts @@ -6,7 +6,7 @@ import type { ElectronFile, PendingUploads, ZipItem } from "../../types/ipc"; import { uploadStatusStore } from "../stores/upload-status"; import { getZipFileStream } from "./fs"; -export const listZipEntries = async (zipPath: string): Promise => { +export const listZipItems = async (zipPath: string): Promise => { const zip = new StreamZip.async({ file: zipPath }); const entries = await zip.entries(); @@ -48,32 +48,34 @@ export const pendingUploads = async (): Promise => { const allFilePaths = uploadStatusStore.get("filePaths") ?? []; const filePaths = allFilePaths.filter((f) => existsSync(f)); - const allZipEntries = uploadStatusStore.get("zipEntries"); - let zipEntries: typeof allZipEntries; + const allZipItems = uploadStatusStore.get("zipItems"); + let zipItems: typeof allZipItems; // Migration code - May 2024. Remove after a bit. // - // The older store formats will not have zipEntries and instead will have + // The older store formats will not have zipItems and instead will have // zipPaths. If we find such a case, read the zipPaths and enqueue all of - // their files as zipEntries in the result. This potentially can be cause us - // to try reuploading an already uploaded file, but the dedup logic will - // kick in at that point so no harm will come off it. - if (allZipEntries === undefined) { + // their files as zipItems in the result. + // + // This potentially can be cause us to try reuploading an already uploaded + // file, but the dedup logic will kick in at that point so no harm will come + // off it. + if (allZipItems === undefined) { const allZipPaths = uploadStatusStore.get("filePaths"); const zipPaths = allZipPaths.filter((f) => existsSync(f)); - zipEntries = []; + zipItems = []; for (const zip of zipPaths) - zipEntries = zipEntries.concat(await listZipEntries(zip)); + zipItems = zipItems.concat(await listZipItems(zip)); } else { - zipEntries = allZipEntries.filter(([z]) => existsSync(z)); + zipItems = allZipItems.filter(([z]) => existsSync(z)); } - if (filePaths.length == 0 && zipEntries.length == 0) return undefined; + if (filePaths.length == 0 && zipItems.length == 0) return undefined; return { collectionName, filePaths, - zipItems: zipEntries, + zipItems, }; }; @@ -86,14 +88,14 @@ export const markUploadedFiles = async (paths: string[]) => { uploadStatusStore.set("filePaths", updated); }; -export const markUploadedZipEntries = async ( - entries: [zipPath: string, entryName: string][], +export const markUploadedZipItems = async ( + items: [zipPath: string, entryName: string][], ) => { - const existing = uploadStatusStore.get("zipEntries"); + const existing = uploadStatusStore.get("zipItems"); const updated = existing.filter( - (z) => !entries.some((e) => z[0] == e[0] && z[1] == e[1]), + (z) => !items.some((e) => z[0] == e[0] && z[1] == e[1]), ); - uploadStatusStore.set("zipEntries", updated); + uploadStatusStore.set("zipItems", updated); }; export const clearPendingUploads = () => uploadStatusStore.clear(); diff --git a/desktop/src/main/stores/upload-status.ts b/desktop/src/main/stores/upload-status.ts index 4fd6c9860..472f38a7f 100644 --- a/desktop/src/main/stores/upload-status.ts +++ b/desktop/src/main/stores/upload-status.ts @@ -21,7 +21,7 @@ export interface UploadStatusStore { */ zipItems?: [zipPath: string, entryName: string][]; /** - * @deprecated Legacy paths to zip files, now subsumed into zipEntries. + * @deprecated Legacy paths to zip files, now subsumed into zipItems. */ zipPaths?: string[]; } diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 48e4d1448..61955b524 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -241,8 +241,8 @@ const watchFindFiles = (folderPath: string): Promise => const pathForFile = (file: File) => webUtils.getPathForFile(file); -const listZipEntries = (zipPath: string): Promise => - ipcRenderer.invoke("listZipEntries", zipPath); +const listZipItems = (zipPath: string): Promise => + ipcRenderer.invoke("listZipItems", zipPath); const pathOrZipItemSize = (pathOrZipItem: string | ZipItem): Promise => ipcRenderer.invoke("pathOrZipItemSize", pathOrZipItem); @@ -256,9 +256,9 @@ const setPendingUploads = (pendingUploads: PendingUploads): Promise => const markUploadedFiles = (paths: PendingUploads["filePaths"]): Promise => ipcRenderer.invoke("markUploadedFiles", paths); -const markUploadedZipEntries = ( - zipEntries: PendingUploads["zipItems"], -): Promise => ipcRenderer.invoke("markUploadedZipEntries", zipEntries); +const markUploadedZipItems = ( + items: PendingUploads["zipItems"], +): Promise => ipcRenderer.invoke("markUploadedZipItems", items); const clearPendingUploads = (): Promise => ipcRenderer.invoke("clearPendingUploads"); @@ -372,11 +372,11 @@ contextBridge.exposeInMainWorld("electron", { // - Upload pathForFile, - listZipEntries, + listZipItems, pathOrZipItemSize, pendingUploads, setPendingUploads, markUploadedFiles, - markUploadedZipEntries, + markUploadedZipItems, clearPendingUploads, }); diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index c895894cc..cd31d2d38 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -1,7 +1,7 @@ import { basename } from "@/next/file"; import log from "@/next/log"; import { type FileAndPath } from "@/next/types/file"; -import type { CollectionMapping, Electron, ZipEntry } from "@/next/types/ipc"; +import type { CollectionMapping, Electron, ZipItem } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; import { isPromise } from "@ente/shared/utils"; import DiscFullIcon from "@mui/icons-material/DiscFull"; @@ -127,15 +127,18 @@ export default function Uploader({ ); /** - * {@link File}s that the user drag-dropped or selected for uploads. This is - * the only type of selection that is possible when we're running in the - * browser. + * {@link File}s that the user drag-dropped or selected for uploads (web). + * + * This is the only type of selection that is possible when we're running in + * the browser. */ const [webFiles, setWebFiles] = useState([]); /** * {@link File}s that the user drag-dropped or selected for uploads, - * augmented with their paths. These siblings of {@link webFiles} come into - * play when we are running in the context of our desktop app. + * augmented with their paths (desktop). + * + * These siblings of {@link webFiles} come into play when we are running in + * the context of our desktop app. */ const [desktopFiles, setDesktopFiles] = useState([]); /** @@ -151,22 +154,24 @@ export default function Uploader({ const [desktopFilePaths, setDesktopFilePaths] = useState([]); /** * (zip file path, entry within zip file) tuples for zip files that the user - * is trying to upload. These are only set when we are running in the - * context of our desktop app. They may be set either on a user action (when - * the user selects or drag-drops zip files) or programmatically (when the - * app is trying to resume pending uploads from a previous session). + * is trying to upload. + * + * These are only set when we are running in the context of our desktop app. + * They may be set either on a user action (when the user selects or + * drag-drops zip files) or programmatically (when the app is trying to + * resume pending uploads from a previous session). */ - const [desktopZipEntries, setDesktopZipEntries] = useState([]); + const [desktopZipItems, setDesktopZipItems] = useState([]); /** * Consolidated and cleaned list obtained from {@link webFiles}, * {@link desktopFiles}, {@link desktopFilePaths} and - * {@link desktopZipEntries}. + * {@link desktopZipItems}. * * Augment each {@link UploadItem} with its "path" (relative path or name in * the case of {@link webFiles}, absolute path in the case of * {@link desktopFiles}, {@link desktopFilePaths}, and the path within the - * zip file for {@link desktopZipEntries}). + * zip file for {@link desktopZipItems}). * * See the documentation of {@link UploadItem} for more details. */ @@ -254,13 +259,13 @@ export default function Uploader({ electron.pendingUploads().then((pending) => { if (!pending) return; - const { collectionName, filePaths, zipEntries } = pending; + const { collectionName, filePaths, zipItems } = pending; log.info("Resuming pending upload", pending); isPendingDesktopUpload.current = true; pendingDesktopUploadCollectionName.current = collectionName; setDesktopFilePaths(filePaths); - setDesktopZipEntries(zipEntries); + setDesktopZipItems(zipItems); }); } }, [ @@ -286,10 +291,10 @@ export default function Uploader({ fileSelectorZipFiles, ].flat(); if (electron) { - desktopFilesAndZipEntries(electron, files).then( - ({ fileAndPaths, zipEntries }) => { + desktopFilesAndZipItems(electron, files).then( + ({ fileAndPaths, zipItems }) => { setDesktopFiles(fileAndPaths); - setDesktopZipEntries(zipEntries); + setDesktopZipItems(zipItems); }, ); } else { @@ -309,7 +314,7 @@ export default function Uploader({ webFiles.map((f) => [f, f["path"] ?? f.name]), desktopFiles.map((fp) => [fp, fp.path]), desktopFilePaths.map((p) => [p, p]), - desktopZipEntries.map((ze) => [ze, ze[1]]), + desktopZipItems.map((ze) => [ze, ze[1]]), ].flat() as [UploadItem, string][]; if (allItemAndPaths.length == 0) return; @@ -333,7 +338,7 @@ export default function Uploader({ setWebFiles([]); setDesktopFiles([]); setDesktopFilePaths([]); - setDesktopZipEntries([]); + setDesktopZipItems([]); // Remove hidden files (files whose names begins with a "."). const prunedItemAndPaths = allItemAndPaths.filter( @@ -423,7 +428,7 @@ export default function Uploader({ intent: CollectionSelectorIntent.upload, }); })(); - }, [webFiles, desktopFiles, desktopFilePaths, desktopZipEntries]); + }, [webFiles, desktopFiles, desktopFilePaths, desktopZipItems]); const preCollectionCreationAction = async () => { props.closeCollectionSelector?.(); @@ -764,23 +769,23 @@ async function waitAndRun( await task(); } -const desktopFilesAndZipEntries = async ( +const desktopFilesAndZipItems = async ( electron: Electron, files: File[], -): Promise<{ fileAndPaths: FileAndPath[]; zipEntries: ZipEntry[] }> => { +): Promise<{ fileAndPaths: FileAndPath[]; zipItems: ZipItem[] }> => { const fileAndPaths: FileAndPath[] = []; - let zipEntries: ZipEntry[] = []; + let zipItems: ZipItem[] = []; for (const file of files) { const path = electron.pathForFile(file); if (file.name.endsWith(".zip")) { - zipEntries = zipEntries.concat(await electron.listZipEntries(path)); + zipItems = zipItems.concat(await electron.listZipItems(path)); } else { fileAndPaths.push({ file, path }); } } - return { fileAndPaths, zipEntries }; + return { fileAndPaths, zipItems }; }; // This is used to prompt the user the make upload strategy choice @@ -891,14 +896,14 @@ export const setPendingUploads = async ( } const filePaths: string[] = []; - const zipEntries: ZipEntry[] = []; + const zipItems: ZipItem[] = []; for (const item of uploadItems) { if (item instanceof File) { throw new Error("Unexpected web file for a desktop pending upload"); } else if (typeof item == "string") { filePaths.push(item); } else if (Array.isArray(item)) { - zipEntries.push(item); + zipItems.push(item); } else { filePaths.push(item.path); } @@ -907,6 +912,6 @@ export const setPendingUploads = async ( await electron.setPendingUploads({ collectionName, filePaths, - zipEntries, + zipItems: zipItems, }); }; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 06772b2d2..44bfef92f 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -4,7 +4,7 @@ import { ensureElectron } from "@/next/electron"; import { lowercaseExtension, nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { type FileAndPath } from "@/next/types/file"; -import type { Electron, ZipEntry } from "@/next/types/ipc"; +import type { Electron, ZipItem } from "@/next/types/ipc"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import { getDedicatedCryptoWorker } from "@ente/shared/crypto"; @@ -105,9 +105,9 @@ const maxConcurrentUploads = 4; * selected the zip file, or it might be a zip file that they'd previously * selected but we now are resuming an interrupted upload. Either ways, what * we have is a path to zip file, and the name of an entry within that zip - * file. This is the {@link ZipEntry} case. + * file. This is the {@link ZipItem} case. */ -export type UploadItem = File | FileAndPath | string | ZipEntry; +export type UploadItem = File | FileAndPath | string | ZipItem; export interface UploadItemWithCollection { localID: number; @@ -840,7 +840,7 @@ const markUploaded = async (electron: Electron, item: ClusteredUploadItem) => { item.livePhotoAssets.video, ]; if (Array.isArray(p0) && Array.isArray(p1)) { - electron.markUploadedZipEntries([p0, p1]); + electron.markUploadedZipItems([p0, p1]); } else if (typeof p0 == "string" && typeof p1 == "string") { electron.markUploadedFiles([p0, p1]); } else if ( @@ -860,7 +860,7 @@ const markUploaded = async (electron: Electron, item: ClusteredUploadItem) => { } else { const p = ensure(item.uploadItem); if (Array.isArray(p)) { - electron.markUploadedZipEntries([p]); + electron.markUploadedZipItems([p]); } else if (typeof p == "string") { electron.markUploadedFiles([p]); } else if (p && typeof p == "object" && "path" in p) { @@ -1030,8 +1030,8 @@ const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => { const uploadItemSize = async (uploadItem: UploadItem): Promise => { if (uploadItem instanceof File) return uploadItem.size; if (typeof uploadItem == "string") - return ensureElectron().pathOrZipEntrySize(uploadItem); + return ensureElectron().pathOrZipItemSize(uploadItem); if (Array.isArray(uploadItem)) - return ensureElectron().pathOrZipEntrySize(uploadItem); + return ensureElectron().pathOrZipItemSize(uploadItem); return uploadItem.file.size; }; diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index c882d5031..d33d41e2f 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -6,7 +6,7 @@ * See: [Note: IPC streams]. */ -import type { Electron, ZipEntry } from "@/next/types/ipc"; +import type { Electron, ZipItem } from "@/next/types/ipc"; /** * Stream the given file or zip entry from the user's local filesystem. @@ -35,7 +35,7 @@ import type { Electron, ZipEntry } from "@/next/types/ipc"; */ export const readStream = async ( _: Electron, - pathOrZipEntry: string | ZipEntry, + pathOrZipEntry: string | ZipItem, ): Promise<{ response: Response; size: number; lastModifiedMs: number }> => { let url: URL; if (typeof pathOrZipEntry == "string") { diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 4b3d97dd3..6efa32f4c 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -221,7 +221,7 @@ export interface Electron { * not yet possible, this function will throw an error with the * {@link CustomErrorMessage.NotAvailable} message. * - * @param dataOrPathOrZipEntry The file whose thumbnail we want to generate. + * @param dataOrPathOrZipItem The file whose thumbnail we want to generate. * It can be provided as raw image data (the contents of the image file), or * the path to the image file, or a tuple containing the path of the zip * file along with the name of an entry in it. @@ -234,14 +234,14 @@ export interface Electron { * @returns JPEG data of the generated thumbnail. */ generateImageThumbnail: ( - dataOrPathOrZipEntry: Uint8Array | string | ZipEntry, + dataOrPathOrZipItem: Uint8Array | string | ZipItem, maxDimension: number, maxSize: number, ) => Promise; /** * Execute a FFmpeg {@link command} on the given - * {@link dataOrPathOrZipEntry}. + * {@link dataOrPathOrZipItem}. * * This executes the command using a FFmpeg executable we bundle with our * desktop app. We also have a wasm FFmpeg wasm implementation that we use @@ -254,7 +254,7 @@ export interface Electron { * (respectively {@link inputPathPlaceholder}, * {@link outputPathPlaceholder}, {@link ffmpegPathPlaceholder}). * - * @param dataOrPathOrZipEntry The bytes of the input file, or the path to + * @param dataOrPathOrZipItem The bytes of the input file, or the path to * the input file on the user's local disk, or the path to a zip file on the * user's disk and the name of an entry in it. In all three cases, the data * gets serialized to a temporary file, and then that path gets substituted @@ -274,7 +274,7 @@ export interface Electron { */ ffmpegExec: ( command: string[], - dataOrPathOrZipEntry: Uint8Array | string | ZipEntry, + dataOrPathOrZipItem: Uint8Array | string | ZipItem, outputFileExtension: string, timeoutMS: number, ) => Promise; @@ -491,13 +491,13 @@ export interface Electron { * * To read the contents of the files themselves, see [Note: IPC streams]. */ - listZipEntries: (zipPath: string) => Promise; + listZipItems: (zipPath: string) => Promise; /** * Return the size in bytes of the file at the given path or of a particular * entry within a zip file. */ - pathOrZipEntrySize: (pathOrZipEntry: string | ZipEntry) => Promise; + pathOrZipItemSize: (pathOrZipItem: string | ZipItem) => Promise; /** * Return any pending uploads that were previously enqueued but haven't yet @@ -518,7 +518,7 @@ export interface Electron { * - Typically, this would be called at the start of an upload. * * - Thereafter, as each item gets uploaded one by one, we'd call - * {@link markUploadedFiles} or {@link markUploadedZipEntries}. + * {@link markUploadedFiles} or {@link markUploadedZipItems}. * * - Finally, once the upload completes (or gets cancelled), we'd call * {@link clearPendingUploads} to complete the circle. @@ -532,11 +532,9 @@ export interface Electron { markUploadedFiles: (paths: PendingUploads["filePaths"]) => Promise; /** - * Mark the given zip file entries as having been uploaded. + * Mark the given {@link ZipItem}s as having been uploaded. */ - markUploadedZipEntries: ( - entries: PendingUploads["zipEntries"], - ) => Promise; + markUploadedZipItems: (items: PendingUploads["zipItems"]) => Promise; /** * Clear any pending uploads. @@ -627,15 +625,17 @@ export interface FolderWatchSyncedFile { } /** - * When the user uploads a zip file, we create a "zip entry" for each entry - * within that zip file. Such an entry is a tuple containin the path to a zip - * file itself, and the name of an entry within it. + * A particular file within a zip file. + * + * When the user uploads a zip file, we create a "zip item" for each entry + * within the zip file. Each such entry is a tuple containing the (path to a zip + * file itself, and the name of an entry within it). * * The name of the entry is not just the file name, but rather is the full path * of the file within the zip. That is, each entry name uniquely identifies a * particular file within the given zip. */ -export type ZipEntry = [zipPath: string, entryName: string]; +export type ZipItem = [zipPath: string, entryName: string]; /** * State about pending and in-progress uploads. @@ -659,7 +659,7 @@ export interface PendingUploads { */ filePaths: string[]; /** - * {@link ZipEntry} (zip path and entry name) that need to be uploaded. + * {@link ZipItem} (zip path and entry name) that need to be uploaded. */ - zipEntries: ZipEntry[]; + zipItems: ZipItem[]; }