0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

web

Browse source
This commit is contained in:
Manav Rathi 2024-04-30 10:11:27 +05:30
parent 73baf5a375
commit afb0e1aff3
No known key found for this signature in database
8 changed files with 95 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
desktop/src/main/ipc.ts
View file

@ -53,9 +53,9 @@ import {
} from "./services/store";
import {
clearPendingUploads,
listZipEntries,
listZipItems,
markUploadedFiles,
markUploadedZipEntries,
markUploadedZipItems,
pathOrZipItemSize,
pendingUploads,
setPendingUploads,
@ -205,8 +205,8 @@ export const attachIPCHandlers = () => {
// - Upload
ipcMain.handle("listZipEntries", (_, zipPath: string) =>
listZipEntries(zipPath),
ipcMain.handle("listZipItems", (_, zipPath: string) =>
listZipItems(zipPath),
);
ipcMain.handle("pathOrZipItemSize", (_, pathOrZipItem: string | ZipItem) =>
@ -225,9 +225,8 @@ export const attachIPCHandlers = () => {
);
ipcMain.handle(
"markUploadedZipEntries",
(_, zipEntries: PendingUploads["zipItems"]) =>
markUploadedZipEntries(zipEntries),
"markUploadedZipItems",
(_, items: PendingUploads["zipItems"]) => markUploadedZipItems(items),
);
ipcMain.handle("clearPendingUploads", () => clearPendingUploads());

38
desktop/src/main/services/upload.ts
View file

@ -6,7 +6,7 @@ import type { ElectronFile, PendingUploads, ZipItem } from "../../types/ipc";
import { uploadStatusStore } from "../stores/upload-status";
import { getZipFileStream } from "./fs";
export const listZipEntries = async (zipPath: string): Promise<ZipItem[]> => {
export const listZipItems = async (zipPath: string): Promise<ZipItem[]> => {
const zip = new StreamZip.async({ file: zipPath });
const entries = await zip.entries();
@ -48,32 +48,34 @@ export const pendingUploads = async (): Promise<PendingUploads | undefined> => {
const allFilePaths = uploadStatusStore.get("filePaths") ?? [];
const filePaths = allFilePaths.filter((f) => existsSync(f));
const allZipEntries = uploadStatusStore.get("zipEntries");
let zipEntries: typeof allZipEntries;
const allZipItems = uploadStatusStore.get("zipItems");
let zipItems: typeof allZipItems;
// Migration code - May 2024. Remove after a bit.
//
// The older store formats will not have zipEntries and instead will have
// The older store formats will not have zipItems and instead will have
// zipPaths. If we find such a case, read the zipPaths and enqueue all of
// their files as zipEntries in the result. This potentially can be cause us
// to try reuploading an already uploaded file, but the dedup logic will
// kick in at that point so no harm will come off it.
if (allZipEntries === undefined) {
// their files as zipItems in the result.
//
// This potentially can be cause us to try reuploading an already uploaded
// file, but the dedup logic will kick in at that point so no harm will come
// off it.
if (allZipItems === undefined) {
const allZipPaths = uploadStatusStore.get("filePaths");
const zipPaths = allZipPaths.filter((f) => existsSync(f));
zipEntries = [];
zipItems = [];
for (const zip of zipPaths)
zipEntries = zipEntries.concat(await listZipEntries(zip));
zipItems = zipItems.concat(await listZipItems(zip));
} else {
zipEntries = allZipEntries.filter(([z]) => existsSync(z));
zipItems = allZipItems.filter(([z]) => existsSync(z));
}
if (filePaths.length == 0 && zipEntries.length == 0) return undefined;
if (filePaths.length == 0 && zipItems.length == 0) return undefined;
return {
collectionName,
filePaths,
zipItems: zipEntries,
zipItems,
};
};
@ -86,14 +88,14 @@ export const markUploadedFiles = async (paths: string[]) => {
uploadStatusStore.set("filePaths", updated);
};
export const markUploadedZipEntries = async (
entries: [zipPath: string, entryName: string][],
export const markUploadedZipItems = async (
items: [zipPath: string, entryName: string][],
) => {
const existing = uploadStatusStore.get("zipEntries");
const existing = uploadStatusStore.get("zipItems");
const updated = existing.filter(
(z) => !entries.some((e) => z[0] == e[0] && z[1] == e[1]),
(z) => !items.some((e) => z[0] == e[0] && z[1] == e[1]),
);
uploadStatusStore.set("zipEntries", updated);
uploadStatusStore.set("zipItems", updated);
};
export const clearPendingUploads = () => uploadStatusStore.clear();

2
desktop/src/main/stores/upload-status.ts
View file

@ -21,7 +21,7 @@ export interface UploadStatusStore {
*/
zipItems?: [zipPath: string, entryName: string][];
/**
* @deprecated Legacy paths to zip files, now subsumed into zipEntries.
* @deprecated Legacy paths to zip files, now subsumed into zipItems.
*/
zipPaths?: string[];
}

14
desktop/src/preload.ts
View file

@ -241,8 +241,8 @@ const watchFindFiles = (folderPath: string): Promise<string[]> =>
const pathForFile = (file: File) => webUtils.getPathForFile(file);
const listZipEntries = (zipPath: string): Promise<ZipItem[]> =>
ipcRenderer.invoke("listZipEntries", zipPath);
const listZipItems = (zipPath: string): Promise<ZipItem[]> =>
ipcRenderer.invoke("listZipItems", zipPath);
const pathOrZipItemSize = (pathOrZipItem: string | ZipItem): Promise<number> =>
ipcRenderer.invoke("pathOrZipItemSize", pathOrZipItem);
@ -256,9 +256,9 @@ const setPendingUploads = (pendingUploads: PendingUploads): Promise<void> =>
const markUploadedFiles = (paths: PendingUploads["filePaths"]): Promise<void> =>
ipcRenderer.invoke("markUploadedFiles", paths);
const markUploadedZipEntries = (
zipEntries: PendingUploads["zipItems"],
): Promise<void> => ipcRenderer.invoke("markUploadedZipEntries", zipEntries);
const markUploadedZipItems = (
items: PendingUploads["zipItems"],
): Promise<void> => ipcRenderer.invoke("markUploadedZipItems", items);
const clearPendingUploads = (): Promise<void> =>
ipcRenderer.invoke("clearPendingUploads");
@ -372,11 +372,11 @@ contextBridge.exposeInMainWorld("electron", {
// - Upload
pathForFile,
listZipEntries,
listZipItems,
pathOrZipItemSize,
pendingUploads,
setPendingUploads,
markUploadedFiles,
markUploadedZipEntries,
markUploadedZipItems,
clearPendingUploads,
});

63
web/apps/photos/src/components/Upload/Uploader.tsx
View file

@ -1,7 +1,7 @@
import { basename } from "@/next/file";
import log from "@/next/log";
import { type FileAndPath } from "@/next/types/file";
import type { CollectionMapping, Electron, ZipEntry } from "@/next/types/ipc";
import type { CollectionMapping, Electron, ZipItem } from "@/next/types/ipc";
import { CustomError } from "@ente/shared/error";
import { isPromise } from "@ente/shared/utils";
import DiscFullIcon from "@mui/icons-material/DiscFull";
@ -127,15 +127,18 @@ export default function Uploader({
);
/**
* {@link File}s that the user drag-dropped or selected for uploads. This is
* the only type of selection that is possible when we're running in the
* browser.
* {@link File}s that the user drag-dropped or selected for uploads (web).
*
* This is the only type of selection that is possible when we're running in
* the browser.
*/
const [webFiles, setWebFiles] = useState<File[]>([]);
/**
* {@link File}s that the user drag-dropped or selected for uploads,
* augmented with their paths. These siblings of {@link webFiles} come into
* play when we are running in the context of our desktop app.
* augmented with their paths (desktop).
*
* These siblings of {@link webFiles} come into play when we are running in
* the context of our desktop app.
*/
const [desktopFiles, setDesktopFiles] = useState<FileAndPath[]>([]);
/**
@ -151,22 +154,24 @@ export default function Uploader({
const [desktopFilePaths, setDesktopFilePaths] = useState<string[]>([]);
/**
* (zip file path, entry within zip file) tuples for zip files that the user
* is trying to upload. These are only set when we are running in the
* context of our desktop app. They may be set either on a user action (when
* the user selects or drag-drops zip files) or programmatically (when the
* app is trying to resume pending uploads from a previous session).
* is trying to upload.
*
* These are only set when we are running in the context of our desktop app.
* They may be set either on a user action (when the user selects or
* drag-drops zip files) or programmatically (when the app is trying to
* resume pending uploads from a previous session).
*/
const [desktopZipEntries, setDesktopZipEntries] = useState<ZipEntry[]>([]);
const [desktopZipItems, setDesktopZipItems] = useState<ZipItem[]>([]);
/**
* Consolidated and cleaned list obtained from {@link webFiles},
* {@link desktopFiles}, {@link desktopFilePaths} and
* {@link desktopZipEntries}.
* {@link desktopZipItems}.
*
* Augment each {@link UploadItem} with its "path" (relative path or name in
* the case of {@link webFiles}, absolute path in the case of
* {@link desktopFiles}, {@link desktopFilePaths}, and the path within the
* zip file for {@link desktopZipEntries}).
* zip file for {@link desktopZipItems}).
*
* See the documentation of {@link UploadItem} for more details.
*/
@ -254,13 +259,13 @@ export default function Uploader({
electron.pendingUploads().then((pending) => {
if (!pending) return;
const { collectionName, filePaths, zipEntries } = pending;
const { collectionName, filePaths, zipItems } = pending;
log.info("Resuming pending upload", pending);
isPendingDesktopUpload.current = true;
pendingDesktopUploadCollectionName.current = collectionName;
setDesktopFilePaths(filePaths);
setDesktopZipEntries(zipEntries);
setDesktopZipItems(zipItems);
});
}
}, [
@ -286,10 +291,10 @@ export default function Uploader({
fileSelectorZipFiles,
].flat();
if (electron) {
desktopFilesAndZipEntries(electron, files).then(
({ fileAndPaths, zipEntries }) => {
desktopFilesAndZipItems(electron, files).then(
({ fileAndPaths, zipItems }) => {
setDesktopFiles(fileAndPaths);
setDesktopZipEntries(zipEntries);
setDesktopZipItems(zipItems);
},
);
} else {
@ -309,7 +314,7 @@ export default function Uploader({
webFiles.map((f) => [f, f["path"] ?? f.name]),
desktopFiles.map((fp) => [fp, fp.path]),
desktopFilePaths.map((p) => [p, p]),
desktopZipEntries.map((ze) => [ze, ze[1]]),
desktopZipItems.map((ze) => [ze, ze[1]]),
].flat() as [UploadItem, string][];
if (allItemAndPaths.length == 0) return;
@ -333,7 +338,7 @@ export default function Uploader({
setWebFiles([]);
setDesktopFiles([]);
setDesktopFilePaths([]);
setDesktopZipEntries([]);
setDesktopZipItems([]);
// Remove hidden files (files whose names begins with a ".").
const prunedItemAndPaths = allItemAndPaths.filter(
@ -423,7 +428,7 @@ export default function Uploader({
intent: CollectionSelectorIntent.upload,
});
})();
}, [webFiles, desktopFiles, desktopFilePaths, desktopZipEntries]);
}, [webFiles, desktopFiles, desktopFilePaths, desktopZipItems]);
const preCollectionCreationAction = async () => {
props.closeCollectionSelector?.();
@ -764,23 +769,23 @@ async function waitAndRun(
await task();
}
const desktopFilesAndZipEntries = async (
const desktopFilesAndZipItems = async (
electron: Electron,
files: File[],
): Promise<{ fileAndPaths: FileAndPath[]; zipEntries: ZipEntry[] }> => {
): Promise<{ fileAndPaths: FileAndPath[]; zipItems: ZipItem[] }> => {
const fileAndPaths: FileAndPath[] = [];
let zipEntries: ZipEntry[] = [];
let zipItems: ZipItem[] = [];
for (const file of files) {
const path = electron.pathForFile(file);
if (file.name.endsWith(".zip")) {
zipEntries = zipEntries.concat(await electron.listZipEntries(path));
zipItems = zipItems.concat(await electron.listZipItems(path));
} else {
fileAndPaths.push({ file, path });
}
}
return { fileAndPaths, zipEntries };
return { fileAndPaths, zipItems };
};
// This is used to prompt the user the make upload strategy choice
@ -891,14 +896,14 @@ export const setPendingUploads = async (
}
const filePaths: string[] = [];
const zipEntries: ZipEntry[] = [];
const zipItems: ZipItem[] = [];
for (const item of uploadItems) {
if (item instanceof File) {
throw new Error("Unexpected web file for a desktop pending upload");
} else if (typeof item == "string") {
filePaths.push(item);
} else if (Array.isArray(item)) {
zipEntries.push(item);
zipItems.push(item);
} else {
filePaths.push(item.path);
}
@ -907,6 +912,6 @@ export const setPendingUploads = async (
await electron.setPendingUploads({
collectionName,
filePaths,
zipEntries,
zipItems: zipItems,
});
};

14
web/apps/photos/src/services/upload/uploadManager.ts
View file

@ -4,7 +4,7 @@ import { ensureElectron } from "@/next/electron";
import { lowercaseExtension, nameAndExtension } from "@/next/file";
import log from "@/next/log";
import { type FileAndPath } from "@/next/types/file";
import type { Electron, ZipEntry } from "@/next/types/ipc";
import type { Electron, ZipItem } from "@/next/types/ipc";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { ensure } from "@/utils/ensure";
import { getDedicatedCryptoWorker } from "@ente/shared/crypto";
@ -105,9 +105,9 @@ const maxConcurrentUploads = 4;
* selected the zip file, or it might be a zip file that they'd previously
* selected but we now are resuming an interrupted upload. Either ways, what
* we have is a path to zip file, and the name of an entry within that zip
* file. This is the {@link ZipEntry} case.
* file. This is the {@link ZipItem} case.
*/
export type UploadItem = File | FileAndPath | string | ZipEntry;
export type UploadItem = File | FileAndPath | string | ZipItem;
export interface UploadItemWithCollection {
localID: number;
@ -840,7 +840,7 @@ const markUploaded = async (electron: Electron, item: ClusteredUploadItem) => {
item.livePhotoAssets.video,
];
if (Array.isArray(p0) && Array.isArray(p1)) {
electron.markUploadedZipEntries([p0, p1]);
electron.markUploadedZipItems([p0, p1]);
} else if (typeof p0 == "string" && typeof p1 == "string") {
electron.markUploadedFiles([p0, p1]);
} else if (
@ -860,7 +860,7 @@ const markUploaded = async (electron: Electron, item: ClusteredUploadItem) => {
} else {
const p = ensure(item.uploadItem);
if (Array.isArray(p)) {
electron.markUploadedZipEntries([p]);
electron.markUploadedZipItems([p]);
} else if (typeof p == "string") {
electron.markUploadedFiles([p]);
} else if (p && typeof p == "object" && "path" in p) {
@ -1030,8 +1030,8 @@ const removePotentialLivePhotoSuffix = (name: string, suffix?: string) => {
const uploadItemSize = async (uploadItem: UploadItem): Promise<number> => {
if (uploadItem instanceof File) return uploadItem.size;
if (typeof uploadItem == "string")
return ensureElectron().pathOrZipEntrySize(uploadItem);
return ensureElectron().pathOrZipItemSize(uploadItem);
if (Array.isArray(uploadItem))
return ensureElectron().pathOrZipEntrySize(uploadItem);
return ensureElectron().pathOrZipItemSize(uploadItem);
return uploadItem.file.size;
};

4
web/apps/photos/src/utils/native-stream.ts
View file

@ -6,7 +6,7 @@
* See: [Note: IPC streams].
*/
import type { Electron, ZipEntry } from "@/next/types/ipc";
import type { Electron, ZipItem } from "@/next/types/ipc";
/**
* Stream the given file or zip entry from the user's local filesystem.
@ -35,7 +35,7 @@ import type { Electron, ZipEntry } from "@/next/types/ipc";
*/
export const readStream = async (
_: Electron,
pathOrZipEntry: string | ZipEntry,
pathOrZipEntry: string | ZipItem,
): Promise<{ response: Response; size: number; lastModifiedMs: number }> => {
let url: URL;
if (typeof pathOrZipEntry == "string") {

36
web/packages/next/types/ipc.ts
View file

@ -221,7 +221,7 @@ export interface Electron {
* not yet possible, this function will throw an error with the
* {@link CustomErrorMessage.NotAvailable} message.
*
* @param dataOrPathOrZipEntry The file whose thumbnail we want to generate.
* @param dataOrPathOrZipItem The file whose thumbnail we want to generate.
* It can be provided as raw image data (the contents of the image file), or
* the path to the image file, or a tuple containing the path of the zip
* file along with the name of an entry in it.
@ -234,14 +234,14 @@ export interface Electron {
* @returns JPEG data of the generated thumbnail.
*/
generateImageThumbnail: (
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
maxDimension: number,
maxSize: number,
) => Promise<Uint8Array>;
/**
* Execute a FFmpeg {@link command} on the given
* {@link dataOrPathOrZipEntry}.
* {@link dataOrPathOrZipItem}.
*
* This executes the command using a FFmpeg executable we bundle with our
* desktop app. We also have a wasm FFmpeg wasm implementation that we use
@ -254,7 +254,7 @@ export interface Electron {
* (respectively {@link inputPathPlaceholder},
* {@link outputPathPlaceholder}, {@link ffmpegPathPlaceholder}).
*
* @param dataOrPathOrZipEntry The bytes of the input file, or the path to
* @param dataOrPathOrZipItem The bytes of the input file, or the path to
* the input file on the user's local disk, or the path to a zip file on the
* user's disk and the name of an entry in it. In all three cases, the data
* gets serialized to a temporary file, and then that path gets substituted
@ -274,7 +274,7 @@ export interface Electron {
*/
ffmpegExec: (
command: string[],
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
outputFileExtension: string,
timeoutMS: number,
) => Promise<Uint8Array>;
@ -491,13 +491,13 @@ export interface Electron {
*
* To read the contents of the files themselves, see [Note: IPC streams].
*/
listZipEntries: (zipPath: string) => Promise<ZipEntry[]>;
listZipItems: (zipPath: string) => Promise<ZipItem[]>;
/**
* Return the size in bytes of the file at the given path or of a particular
* entry within a zip file.
*/
pathOrZipEntrySize: (pathOrZipEntry: string | ZipEntry) => Promise<number>;
pathOrZipItemSize: (pathOrZipItem: string | ZipItem) => Promise<number>;
/**
* Return any pending uploads that were previously enqueued but haven't yet
@ -518,7 +518,7 @@ export interface Electron {
* - Typically, this would be called at the start of an upload.
*
* - Thereafter, as each item gets uploaded one by one, we'd call
* {@link markUploadedFiles} or {@link markUploadedZipEntries}.
* {@link markUploadedFiles} or {@link markUploadedZipItems}.
*
* - Finally, once the upload completes (or gets cancelled), we'd call
* {@link clearPendingUploads} to complete the circle.
@ -532,11 +532,9 @@ export interface Electron {
markUploadedFiles: (paths: PendingUploads["filePaths"]) => Promise<void>;
/**
* Mark the given zip file entries as having been uploaded.
* Mark the given {@link ZipItem}s as having been uploaded.
*/
markUploadedZipEntries: (
entries: PendingUploads["zipEntries"],
) => Promise<void>;
markUploadedZipItems: (items: PendingUploads["zipItems"]) => Promise<void>;
/**
* Clear any pending uploads.
@ -627,15 +625,17 @@ export interface FolderWatchSyncedFile {
}
/**
* When the user uploads a zip file, we create a "zip entry" for each entry
* within that zip file. Such an entry is a tuple containin the path to a zip
* file itself, and the name of an entry within it.
* A particular file within a zip file.
*
* When the user uploads a zip file, we create a "zip item" for each entry
* within the zip file. Each such entry is a tuple containing the (path to a zip
* file itself, and the name of an entry within it).
*
* The name of the entry is not just the file name, but rather is the full path
* of the file within the zip. That is, each entry name uniquely identifies a
* particular file within the given zip.
*/
export type ZipEntry = [zipPath: string, entryName: string];
export type ZipItem = [zipPath: string, entryName: string];
/**
* State about pending and in-progress uploads.
@ -659,7 +659,7 @@ export interface PendingUploads {
*/
filePaths: string[];
/**
* {@link ZipEntry} (zip path and entry name) that need to be uploaded.
* {@link ZipItem} (zip path and entry name) that need to be uploaded.
*/
zipEntries: ZipEntry[];
zipItems: ZipItem[];
}