0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Uncollide with ZipEntry from StreamZip

Browse source
This commit is contained in:
Manav Rathi 2024-04-30 10:08:31 +05:30
parent e9bf26e421
commit 73baf5a375
No known key found for this signature in database
9 changed files with 75 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
desktop/src/main/ipc.ts
View file

@ -14,7 +14,7 @@ import type {
CollectionMapping,
FolderWatch,
PendingUploads,
ZipEntry,
ZipItem,
} from "../types/ipc";
import {
selectDirectory,
@ -56,7 +56,7 @@ import {
listZipEntries,
markUploadedFiles,
markUploadedZipEntries,
pathOrZipEntrySize,
pathOrZipItemSize,
pendingUploads,
setPendingUploads,
} from "./services/upload";
@ -152,11 +152,10 @@ export const attachIPCHandlers = () => {
"generateImageThumbnail",
(
_,
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
maxDimension: number,
maxSize: number,
) =>
generateImageThumbnail(dataOrPathOrZipEntry, maxDimension, maxSize),
) => generateImageThumbnail(dataOrPathOrZipItem, maxDimension, maxSize),
);
ipcMain.handle(
@ -164,13 +163,13 @@ export const attachIPCHandlers = () => {
(
_,
command: string[],
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
outputFileExtension: string,
timeoutMS: number,
) =>
ffmpegExec(
command,
dataOrPathOrZipEntry,
dataOrPathOrZipItem,
outputFileExtension,
timeoutMS,
),
@ -210,10 +209,8 @@ export const attachIPCHandlers = () => {
listZipEntries(zipPath),
);
ipcMain.handle(
"pathOrZipEntrySize",
(_, pathOrZipEntry: string | ZipEntry) =>
pathOrZipEntrySize(pathOrZipEntry),
ipcMain.handle("pathOrZipItemSize", (_, pathOrZipItem: string | ZipItem) =>
pathOrZipItemSize(pathOrZipItem),
);
ipcMain.handle("pendingUploads", () => pendingUploads());
@ -229,7 +226,7 @@ export const attachIPCHandlers = () => {
ipcMain.handle(
"markUploadedZipEntries",
(_, zipEntries: PendingUploads["zipEntries"]) =>
(_, zipEntries: PendingUploads["zipItems"]) =>
markUploadedZipEntries(zipEntries),
);

10
desktop/src/main/services/ffmpeg.ts
View file

@ -1,12 +1,12 @@
import pathToFfmpeg from "ffmpeg-static";
import fs from "node:fs/promises";
import type { ZipEntry } from "../../types/ipc";
import type { ZipItem } from "../../types/ipc";
import log from "../log";
import { withTimeout } from "../utils";
import { execAsync } from "../utils-electron";
import {
deleteTempFile,
makeFileForDataOrPathOrZipEntry,
makeFileForDataOrPathOrZipItem,
makeTempFilePath,
} from "../utils-temp";
@ -44,12 +44,12 @@ const outputPathPlaceholder = "OUTPUT";
*/
export const ffmpegExec = async (
command: string[],
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
outputFileExtension: string,
timeoutMS: number,
): Promise<Uint8Array> => {
// TODO (MR): This currently copies files for both input (when
// dataOrPathOrZipEntry is data) and output. This needs to be tested
// dataOrPathOrZipItem is data) and output. This needs to be tested
// extremely large video files when invoked downstream of `convertToMP4` in
// the web code.
@ -57,7 +57,7 @@ export const ffmpegExec = async (
path: inputFilePath,
isFileTemporary: isInputFileTemporary,
writeToTemporaryFile: writeToTemporaryInputFile,
} = await makeFileForDataOrPathOrZipEntry(dataOrPathOrZipEntry);
} = await makeFileForDataOrPathOrZipItem(dataOrPathOrZipItem);
const outputFilePath = await makeTempFilePath(outputFileExtension);
try {

8
desktop/src/main/services/image.ts
View file

@ -2,12 +2,12 @@
import fs from "node:fs/promises";
import path from "path";
import { CustomErrorMessage, type ZipEntry } from "../../types/ipc";
import { CustomErrorMessage, type ZipItem } from "../../types/ipc";
import log from "../log";
import { execAsync, isDev } from "../utils-electron";
import {
deleteTempFile,
makeFileForDataOrPathOrZipEntry,
makeFileForDataOrPathOrZipItem,
makeTempFilePath,
} from "../utils-temp";
@ -67,7 +67,7 @@ const imageMagickPath = () =>
path.join(isDev ? "build" : process.resourcesPath, "image-magick");
export const generateImageThumbnail = async (
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
maxDimension: number,
maxSize: number,
): Promise<Uint8Array> => {
@ -75,7 +75,7 @@ export const generateImageThumbnail = async (
path: inputFilePath,
isFileTemporary: isInputFileTemporary,
writeToTemporaryFile: writeToTemporaryInputFile,
} = await makeFileForDataOrPathOrZipEntry(dataOrPathOrZipEntry);
} = await makeFileForDataOrPathOrZipItem(dataOrPathOrZipItem);
const outputFilePath = await makeTempFilePath("jpeg");

16
desktop/src/main/services/upload.ts
View file

@ -2,11 +2,11 @@ import StreamZip from "node-stream-zip";
import fs from "node:fs/promises";
import { existsSync } from "original-fs";
import path from "path";
import type { ElectronFile, PendingUploads, ZipEntry } from "../../types/ipc";
import type { ElectronFile, PendingUploads, ZipItem } from "../../types/ipc";
import { uploadStatusStore } from "../stores/upload-status";
import { getZipFileStream } from "./fs";
export const listZipEntries = async (zipPath: string): Promise<ZipEntry[]> => {
export const listZipEntries = async (zipPath: string): Promise<ZipItem[]> => {
const zip = new StreamZip.async({ file: zipPath });
const entries = await zip.entries();
@ -26,14 +26,14 @@ export const listZipEntries = async (zipPath: string): Promise<ZipEntry[]> => {
return entryNames.map((entryName) => [zipPath, entryName]);
};
export const pathOrZipEntrySize = async (
pathOrZipEntry: string | ZipEntry,
export const pathOrZipItemSize = async (
pathOrZipItem: string | ZipItem,
): Promise<number> => {
if (typeof pathOrZipEntry == "string") {
const stat = await fs.stat(pathOrZipEntry);
if (typeof pathOrZipItem == "string") {
const stat = await fs.stat(pathOrZipItem);
return stat.size;
} else {
const [zipPath, entryName] = pathOrZipEntry;
const [zipPath, entryName] = pathOrZipItem;
const zip = new StreamZip.async({ file: zipPath });
const entry = await zip.entry(entryName);
const size = entry.size;
@ -73,7 +73,7 @@ export const pendingUploads = async (): Promise<PendingUploads | undefined> => {
return {
collectionName,
filePaths,
zipEntries,
zipItems: zipEntries,
};
};

21
desktop/src/main/stores/upload-status.ts
View file

@ -1,17 +1,28 @@
import Store, { Schema } from "electron-store";
export interface UploadStatusStore {
/* The collection to which we're uploading, or the root collection. */
/**
* The collection to which we're uploading, or the root collection.
*
* Not all pending uploads will have an associated collection.
*/
collectionName?: string;
/** Paths to regular files that are pending upload */
/**
* Paths to regular files that are pending upload.
*
* This should generally be present, albeit empty, but it is marked optional
* in sympathy with its siblings.
*/
filePaths?: string[];
/**
* Each item is the path to a zip file and the name of an entry within it.
*
* This is marked optional since legacy stores will not have it.
*/
zipEntries?: [zipPath: string, entryName: string][];
/** Legacy paths to zip files, now subsumed into zipEntries */
zipItems?: [zipPath: string, entryName: string][];
/**
* @deprecated Legacy paths to zip files, now subsumed into zipEntries.
*/
zipPaths?: string[];
}
@ -25,7 +36,7 @@ const uploadStatusSchema: Schema<UploadStatusStore> = {
type: "string",
},
},
zipEntries: {
zipItems: {
type: "array",
items: {
type: "array",

6
desktop/src/main/stream.ts
View file

@ -95,10 +95,10 @@ const handleRead = async (path: string) => {
}
};
const handleReadZip = async (zipPath: string, zipEntryPath: string) => {
const handleReadZip = async (zipPath: string, entryName: string) => {
try {
const zip = new StreamZip.async({ file: zipPath });
const entry = await zip.entry(zipEntryPath);
const entry = await zip.entry(entryName);
const stream = await zip.stream(entry);
// TODO(MR): when to call zip.close()
@ -119,7 +119,7 @@ const handleReadZip = async (zipPath: string, zipEntryPath: string) => {
});
} catch (e) {
log.error(
`Failed to read entry ${zipEntryPath} from zip file at ${zipPath}`,
`Failed to read entry ${entryName} from zip file at ${zipPath}`,
e,
);
return new Response(`Failed to read stream: ${e.message}`, {

34
desktop/src/main/utils-temp.ts
View file

@ -3,7 +3,7 @@ import StreamZip from "node-stream-zip";
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import path from "path";
import type { ZipEntry } from "../types/ipc";
import type { ZipItem } from "../types/ipc";
/**
* Our very own directory within the system temp directory. Go crazy, but
@ -64,9 +64,11 @@ export const deleteTempFile = async (tempFilePath: string) => {
await fs.rm(tempFilePath, { force: true });
};
/** The result of {@link makeFileForDataOrPathOrZipEntry}. */
interface FileForDataOrPathOrZipEntry {
/** The path to the file (possibly temporary) */
/** The result of {@link makeFileForDataOrPathOrZipItem}. */
interface FileForDataOrPathOrZipItem {
/**
* The path to the file (possibly temporary).
*/
path: string;
/**
* `true` if {@link path} points to a temporary file which should be deleted
@ -75,12 +77,12 @@ interface FileForDataOrPathOrZipEntry {
isFileTemporary: boolean;
/**
* If set, this'll be a function that can be called to actually write the
* contents of the source `Uint8Array | string | ZipEntry` into the file at
* contents of the source `Uint8Array | string | ZipItem` into the file at
* {@link path}.
*
* It will be undefined if the source is already a path since nothing needs
* to be written in that case. In the other two cases this function will
* write the data or zip entry into the file at {@link path}.
* write the data or zip item into the file at {@link path}.
*/
writeToTemporaryFile?: () => Promise<void>;
}
@ -88,31 +90,31 @@ interface FileForDataOrPathOrZipEntry {
/**
* Return the path to a file, a boolean indicating if this is a temporary path
* that needs to be deleted after processing, and a function to write the given
* {@link dataOrPathOrZipEntry} into that temporary file if needed.
* {@link dataOrPathOrZipItem} into that temporary file if needed.
*
* @param dataOrPathOrZipEntry The contents of the file, or the path to an
* @param dataOrPathOrZipItem The contents of the file, or the path to an
* existing file, or a (path to a zip file, name of an entry within that zip
* file) tuple.
*/
export const makeFileForDataOrPathOrZipEntry = async (
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
): Promise<FileForDataOrPathOrZipEntry> => {
export const makeFileForDataOrPathOrZipItem = async (
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
): Promise<FileForDataOrPathOrZipItem> => {
let path: string;
let isFileTemporary: boolean;
let writeToTemporaryFile: () => Promise<void> | undefined;
if (typeof dataOrPathOrZipEntry == "string") {
path = dataOrPathOrZipEntry;
if (typeof dataOrPathOrZipItem == "string") {
path = dataOrPathOrZipItem;
isFileTemporary = false;
} else {
path = await makeTempFilePath();
isFileTemporary = true;
if (dataOrPathOrZipEntry instanceof Uint8Array) {
if (dataOrPathOrZipItem instanceof Uint8Array) {
writeToTemporaryFile = () =>
fs.writeFile(path, dataOrPathOrZipEntry);
fs.writeFile(path, dataOrPathOrZipItem);
} else {
writeToTemporaryFile = async () => {
const [zipPath, entryName] = dataOrPathOrZipEntry;
const [zipPath, entryName] = dataOrPathOrZipItem;
const zip = new StreamZip.async({ file: zipPath });
await zip.extract(entryName, path);
zip.close();

21
desktop/src/preload.ts
View file

@ -47,7 +47,7 @@ import type {
ElectronFile,
FolderWatch,
PendingUploads,
ZipEntry,
ZipItem,
} from "./types/ipc";
// - General
@ -129,27 +129,27 @@ const convertToJPEG = (imageData: Uint8Array): Promise<Uint8Array> =>
ipcRenderer.invoke("convertToJPEG", imageData);
const generateImageThumbnail = (
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
maxDimension: number,
maxSize: number,
): Promise<Uint8Array> =>
ipcRenderer.invoke(
"generateImageThumbnail",
dataOrPathOrZipEntry,
dataOrPathOrZipItem,
maxDimension,
maxSize,
);
const ffmpegExec = (
command: string[],
dataOrPathOrZipEntry: Uint8Array | string | ZipEntry,
dataOrPathOrZipItem: Uint8Array | string | ZipItem,
outputFileExtension: string,
timeoutMS: number,
): Promise<Uint8Array> =>
ipcRenderer.invoke(
"ffmpegExec",
command,
dataOrPathOrZipEntry,
dataOrPathOrZipItem,
outputFileExtension,
timeoutMS,
);
@ -241,12 +241,11 @@ const watchFindFiles = (folderPath: string): Promise<string[]> =>
const pathForFile = (file: File) => webUtils.getPathForFile(file);
const listZipEntries = (zipPath: string): Promise<ZipEntry[]> =>
const listZipEntries = (zipPath: string): Promise<ZipItem[]> =>
ipcRenderer.invoke("listZipEntries", zipPath);
const pathOrZipEntrySize = (
pathOrZipEntry: string | ZipEntry,
): Promise<number> => ipcRenderer.invoke("pathOrZipEntrySize", pathOrZipEntry);
const pathOrZipItemSize = (pathOrZipItem: string | ZipItem): Promise<number> =>
ipcRenderer.invoke("pathOrZipItemSize", pathOrZipItem);
const pendingUploads = (): Promise<PendingUploads | undefined> =>
ipcRenderer.invoke("pendingUploads");
@ -258,7 +257,7 @@ const markUploadedFiles = (paths: PendingUploads["filePaths"]): Promise<void> =>
ipcRenderer.invoke("markUploadedFiles", paths);
const markUploadedZipEntries = (
zipEntries: PendingUploads["zipEntries"],
zipEntries: PendingUploads["zipItems"],
): Promise<void> => ipcRenderer.invoke("markUploadedZipEntries", zipEntries);
const clearPendingUploads = (): Promise<void> =>
@ -374,7 +373,7 @@ contextBridge.exposeInMainWorld("electron", {
pathForFile,
listZipEntries,
pathOrZipEntrySize,
pathOrZipItemSize,
pendingUploads,
setPendingUploads,
markUploadedFiles,

4
desktop/src/types/ipc.ts
View file

@ -25,12 +25,12 @@ export interface FolderWatchSyncedFile {
collectionID: number;
}
export type ZipEntry = [zipPath: string, entryName: string];
export type ZipItem = [zipPath: string, entryName: string];
export interface PendingUploads {
collectionName: string;
filePaths: string[];
zipEntries: ZipEntry[];
zipItems: ZipItem[];
}
/**