diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 271577aa0..d7d4fdc09 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -53,11 +53,11 @@ import { } from "./services/store"; import { clearPendingUploads, - lsZip, markUploadedFiles, markUploadedZipEntries, pendingUploads, setPendingUploads, + zipEntries, } from "./services/upload"; import { watchAdd, @@ -200,7 +200,7 @@ export const attachIPCHandlers = () => { // - Upload - ipcMain.handle("lsZip", (_, zipPath: string) => lsZip(zipPath)); + ipcMain.handle("zipEntries", (_, zipPath: string) => zipEntries(zipPath)); ipcMain.handle("pendingUploads", () => pendingUploads()); diff --git a/desktop/src/main/services/upload.ts b/desktop/src/main/services/upload.ts index ebd1f481f..c5a987e7b 100644 --- a/desktop/src/main/services/upload.ts +++ b/desktop/src/main/services/upload.ts @@ -1,35 +1,53 @@ import StreamZip from "node-stream-zip"; import { existsSync } from "original-fs"; import path from "path"; -import { ElectronFile, type PendingUploads } from "../../types/ipc"; +import type { ElectronFile, PendingUploads, ZipEntry } from "../../types/ipc"; import { uploadStatusStore, type UploadStatusStore, } from "../stores/upload-status"; import { getElectronFile, getZipFileStream } from "./fs"; -export const lsZip = async (zipPath: string) => { +export const zipEntries = async (zipPath: string): Promise => { const zip = new StreamZip.async({ file: zipPath }); const entries = await zip.entries(); - const entryPaths: string[] = []; + const entryNames: string[] = []; for (const entry of Object.values(entries)) { const basename = path.basename(entry.name); // Ignore "hidden" files (files whose names begins with a dot). if (entry.isFile && basename.length > 0 && basename[0] != ".") { // `entry.name` is the path within the zip. - entryPaths.push(entry.name); + entryNames.push(entry.name); } } - return [entryPaths]; + return entryNames.map((entryName) => [zipPath, entryName]); }; export const pendingUploads = async (): Promise => { - /* TODO */ const collectionName = uploadStatusStore.get("collectionName"); - const filePaths = validSavedPaths("files"); + if (!collectionName) return undefined; + + const allFilePaths = uploadStatusStore.get("filePaths"); + const filePaths = allFilePaths.filter((f) => existsSync(f)); + + let allZipEntries = uploadStatusStore.get("zipEntries"); + // Migration code - May 2024. Remove after a bit. + // + // The older store formats will not have zipEntries and instead will have + // zipPaths. If we find such a case, read the zipPaths and enqueue all of + // their files as zipEntries in the result. This potentially can be cause us + // to try reuploading an already uploaded file, but the dedup logic will + // kick in at that point so no harm will come off it. + if (allZipEntries === undefined) { + const allZipPaths = uploadStatusStore.get("filePaths"); + const zipPaths = allZipPaths.filter((f) => existsSync(f)); + lsZip(); + } + + if (allZipEntries) "files"; const zipPaths = validSavedPaths("zips"); let files: ElectronFile[] = []; diff --git a/desktop/src/main/stores/upload-status.ts b/desktop/src/main/stores/upload-status.ts index 36a7d1fa7..edd086fbe 100644 --- a/desktop/src/main/stores/upload-status.ts +++ b/desktop/src/main/stores/upload-status.ts @@ -7,8 +7,10 @@ export interface UploadStatusStore { filePaths: string[]; /** * Each item is the path to a zip file and the name of an entry within it. + * + * This is marked optional since legacy stores will not have it. */ - zipEntries: [zipPath: string, entryName: string][]; + zipEntries?: [zipPath: string, entryName: string][]; /** Legacy paths to zip files, now subsumed into zipEntries */ zipPaths?: string[]; } diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 484a3bc0e..ac149ad13 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -47,6 +47,7 @@ import type { ElectronFile, FolderWatch, PendingUploads, + ZipEntry, } from "./types/ipc"; // - General @@ -241,8 +242,8 @@ const watchFindFiles = (folderPath: string): Promise => // - Upload -const lsZip = (zipPath: string): Promise => - ipcRenderer.invoke("lsZip", zipPath); +const zipEntries = (zipPath: string): Promise => + ipcRenderer.invoke("zipEntries", zipPath); const pendingUploads = (): Promise => ipcRenderer.invoke("pendingUploads"); @@ -369,7 +370,7 @@ contextBridge.exposeInMainWorld("electron", { // - Upload - lsZip, + zipEntries, pendingUploads, setPendingUploads, markUploadedFiles, diff --git a/desktop/src/types/ipc.ts b/desktop/src/types/ipc.ts index f343e2bba..307fb7de3 100644 --- a/desktop/src/types/ipc.ts +++ b/desktop/src/types/ipc.ts @@ -25,10 +25,12 @@ export interface FolderWatchSyncedFile { collectionID: number; } +export type ZipEntry = [zipPath: string, entryName: string]; + export interface PendingUploads { collectionName: string; filePaths: string[]; - zipEntries: [zipPath: string, entryName: string][]; + zipEntries: ZipEntry[]; } /** diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 761cd72f1..6aa394c6c 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -470,19 +470,22 @@ export interface Electron { * * @param zipPath The path of the zip file on the user's local file system. * - * @returns A list of paths, one for each file in the given zip. Directories - * are traversed recursively, but the directory entries themselves will be - * excluded from the returned list. File entries whose file name begins with - * a dot (i.e. "hidden" files) will also be excluded. + * @returns A list of (zipPath, entryName) tuples, one for each file in the + * given zip. Directories are traversed recursively, but the directory + * entries themselves will be excluded from the returned list. File entries + * whose file name begins with a dot (i.e. "hidden" files) will also be + * excluded. * * To read the contents of the files themselves, see [Note: IPC streams]. */ - lsZip: (zipPath: string) => Promise; + zipEntries : (zipPath: string) => Promise /** * Return any pending uploads that were previously enqueued but haven't yet * been completed. * + * Return undefined if there are no such pending uploads. + * * The state of pending uploads is persisted in the Node.js layer. Or app * start, we read in this data from the Node.js layer via this IPC method. * The Node.js code returns the persisted data after filtering out any files @@ -493,10 +496,13 @@ export interface Electron { /** * Set the state of pending uploads. * - * Typically, this would be called at the start of an upload. Thereafter, as - * each item gets uploaded one by one, we'd call {@link markUploaded}. - * Finally, once the upload completes (or gets cancelled), we'd call - * {@link clearPendingUploads} to complete the circle. + * - Typically, this would be called at the start of an upload. + * + * - Thereafter, as each item gets uploaded one by one, we'd call + * {@link markUploadedFiles} or {@link markUploadedZipEntries}. + * + * - Finally, once the upload completes (or gets cancelled), we'd call + * {@link clearPendingUploads} to complete the circle. */ setPendingUploads: (pendingUploads: PendingUploads) => Promise; @@ -601,6 +607,17 @@ export interface FolderWatchSyncedFile { collectionID: number; } +/** + * When the user uploads a zip file, we create a "zip entry" for each entry + * within that zip file. Such an entry is a tuple containin the path to a zip + * file itself, and the name of an entry within it. + * + * The name of the entry is not just the file name, but rather is the full path + * of the file within the zip. That is, each entry name uniquely identifies a + * particular file within the given zip. + */ +export type ZipEntry = [zipPath: string, entryName: string]; + /** * State about pending and in-progress uploads. * @@ -623,12 +640,7 @@ export interface PendingUploads { */ filePaths: string[]; /** - * When the user uploads a zip file, we create a "zip entry" for each entry - * within that zip file. Such an entry is a tuple containin the path to a - * zip file itself, and the name of an entry within it. - * - * These are the remaining of those zip entries that still need to be - * uploaded. + * {@link ZipEntry} (zip path and entry name) that need to be uploaded. */ - zipEntries: [zipPath: string, entryName: string][]; + zipEntries: ZipEntry[]; }