0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Bye ElectronFile

Browse source
This commit is contained in:
Manav Rathi 2024-04-30 13:26:55 +05:30
parent f5754eb2e1
commit f84937f8c1
No known key found for this signature in database
12 changed files with 44 additions and 351 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
desktop/src/main/dialogs.ts
View file

@ -1,72 +0,0 @@
import { dialog } from "electron/main";
import fs from "node:fs/promises";
import path from "node:path";
import type { ElectronFile } from "../types/ipc";
import { getElectronFile } from "./services/fs";
import { getElectronFilesFromGoogleZip } from "./services/upload";
export const selectDirectory = async () => {
const result = await dialog.showOpenDialog({
properties: ["openDirectory"],
});
if (result.filePaths && result.filePaths.length > 0) {
return result.filePaths[0]?.split(path.sep)?.join(path.posix.sep);
}
};
export const showUploadFilesDialog = async () => {
const selectedFiles = await dialog.showOpenDialog({
properties: ["openFile", "multiSelections"],
});
const filePaths = selectedFiles.filePaths;
return await Promise.all(filePaths.map(getElectronFile));
};
export const showUploadDirsDialog = async () => {
const dir = await dialog.showOpenDialog({
properties: ["openDirectory", "multiSelections"],
});
let filePaths: string[] = [];
for (const dirPath of dir.filePaths) {
filePaths = [...filePaths, ...(await getDirFilePaths(dirPath))];
}
return await Promise.all(filePaths.map(getElectronFile));
};
// https://stackoverflow.com/a/63111390
const getDirFilePaths = async (dirPath: string) => {
if (!(await fs.stat(dirPath)).isDirectory()) {
return [dirPath];
}
let files: string[] = [];
const filePaths = await fs.readdir(dirPath);
for (const filePath of filePaths) {
const absolute = path.join(dirPath, filePath);
files = [...files, ...(await getDirFilePaths(absolute))];
}
return files;
};
export const showUploadZipDialog = async () => {
const selectedFiles = await dialog.showOpenDialog({
properties: ["openFile", "multiSelections"],
filters: [{ name: "Zip File", extensions: ["zip"] }],
});
const filePaths = selectedFiles.filePaths;
let files: ElectronFile[] = [];
for (const filePath of filePaths) {
files = [...files, ...(await getElectronFilesFromGoogleZip(filePath))];
}
return {
zipPaths: filePaths,
files,
};
};

19
desktop/src/main/ipc.ts
View file

@ -16,12 +16,6 @@ import type {
PendingUploads,
ZipItem,
} from "../types/ipc";
import {
selectDirectory,
showUploadDirsDialog,
showUploadFilesDialog,
showUploadZipDialog,
} from "./dialogs";
import {
fsExists,
fsIsDir,
@ -39,6 +33,7 @@ import {
updateAndRestart,
updateOnNextRestart,
} from "./services/app-update";
import { selectDirectory } from "./services/dialog";
import { ffmpegExec } from "./services/ffmpeg";
import { convertToJPEG, generateImageThumbnail } from "./services/image";
import {
@ -102,6 +97,8 @@ export const attachIPCHandlers = () => {
// See [Note: Catching exception during .send/.on]
ipcMain.on("logToDisk", (_, message) => logToDisk(message));
ipcMain.handle("selectDirectory", () => selectDirectory());
ipcMain.on("clearStores", () => clearStores());
ipcMain.handle("saveEncryptionKey", (_, encryptionKey) =>
@ -193,16 +190,6 @@ export const attachIPCHandlers = () => {
faceEmbedding(input),
);
// - File selection
ipcMain.handle("selectDirectory", () => selectDirectory());
ipcMain.handle("showUploadFilesDialog", () => showUploadFilesDialog());
ipcMain.handle("showUploadDirsDialog", () => showUploadDirsDialog());
ipcMain.handle("showUploadZipDialog", () => showUploadZipDialog());
// - Upload
ipcMain.handle("listZipItems", (_, zipPath: string) =>

10
desktop/src/main/services/dialog.ts Normal file
View file

@ -0,0 +1,10 @@
import { dialog } from "electron/main";
import { posixPath } from "../utils-path";
export const selectDirectory = async () => {
const result = await dialog.showOpenDialog({
properties: ["openDirectory"],
});
const dirPath = result.filePaths[0];
return dirPath ? posixPath(dirPath) : undefined;
};

154
desktop/src/main/services/fs.ts
View file

@ -1,154 +0,0 @@
import StreamZip from "node-stream-zip";
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { ElectronFile } from "../../types/ipc";
import log from "../log";
const FILE_STREAM_CHUNK_SIZE: number = 4 * 1024 * 1024;
const getFileStream = async (filePath: string) => {
const file = await fs.open(filePath, "r");
let offset = 0;
const readableStream = new ReadableStream<Uint8Array>({
async pull(controller) {
try {
const buff = new Uint8Array(FILE_STREAM_CHUNK_SIZE);
const bytesRead = (await file.read(
buff,
0,
FILE_STREAM_CHUNK_SIZE,
offset,
)) as unknown as number;
offset += bytesRead;
if (bytesRead === 0) {
controller.close();
await file.close();
} else {
controller.enqueue(buff.slice(0, bytesRead));
}
} catch (e) {
await file.close();
}
},
async cancel() {
await file.close();
},
});
return readableStream;
};
export async function getElectronFile(filePath: string): Promise<ElectronFile> {
const fileStats = await fs.stat(filePath);
return {
path: filePath.split(path.sep).join(path.posix.sep),
name: path.basename(filePath),
size: fileStats.size,
lastModified: fileStats.mtime.valueOf(),
stream: async () => {
if (!existsSync(filePath)) {
throw new Error("electronFile does not exist");
}
return await getFileStream(filePath);
},
blob: async () => {
if (!existsSync(filePath)) {
throw new Error("electronFile does not exist");
}
const blob = await fs.readFile(filePath);
return new Blob([new Uint8Array(blob)]);
},
arrayBuffer: async () => {
if (!existsSync(filePath)) {
throw new Error("electronFile does not exist");
}
const blob = await fs.readFile(filePath);
return new Uint8Array(blob);
},
};
}
export const getZipFileStream = async (
zip: StreamZip.StreamZipAsync,
filePath: string,
) => {
const stream = await zip.stream(filePath);
const done = {
current: false,
};
const inProgress = {
current: false,
};
// eslint-disable-next-line no-unused-vars
let resolveObj: (value?: any) => void = null;
// eslint-disable-next-line no-unused-vars
let rejectObj: (reason?: any) => void = null;
stream.on("readable", () => {
try {
if (resolveObj) {
inProgress.current = true;
const chunk = stream.read(FILE_STREAM_CHUNK_SIZE) as Buffer;
if (chunk) {
resolveObj(new Uint8Array(chunk));
resolveObj = null;
}
inProgress.current = false;
}
} catch (e) {
rejectObj(e);
}
});
stream.on("end", () => {
try {
done.current = true;
if (resolveObj && !inProgress.current) {
resolveObj(null);
resolveObj = null;
}
} catch (e) {
rejectObj(e);
}
});
stream.on("error", (e) => {
try {
done.current = true;
if (rejectObj) {
rejectObj(e);
rejectObj = null;
}
} catch (e) {
rejectObj(e);
}
});
const readStreamData = async () => {
return new Promise<Uint8Array>((resolve, reject) => {
const chunk = stream.read(FILE_STREAM_CHUNK_SIZE) as Buffer;
if (chunk || done.current) {
resolve(chunk);
} else {
resolveObj = resolve;
rejectObj = reject;
}
});
};
const readableStream = new ReadableStream<Uint8Array>({
async pull(controller) {
try {
const data = await readStreamData();
if (data) {
controller.enqueue(data);
} else {
controller.close();
}
} catch (e) {
log.error("Failed to pull from readableStream", e);
controller.close();
}
},
});
return readableStream;
};

2
desktop/src/main/services/image.ts
View file

@ -1,7 +1,7 @@
/** @file Image format conversions and thumbnail generation */
import fs from "node:fs/promises";
import path from "path";
import path from "node:path";
import { CustomErrorMessage, type ZipItem } from "../../types/ipc";
import log from "../log";
import { execAsync, isDev } from "../utils-electron";

53
desktop/src/main/services/upload.ts
View file

@ -1,10 +1,9 @@
import StreamZip from "node-stream-zip";
import fs from "node:fs/promises";
import path from "node:path";
import { existsSync } from "original-fs";
import path from "path";
import type { ElectronFile, PendingUploads, ZipItem } from "../../types/ipc";
import type { PendingUploads, ZipItem } from "../../types/ipc";
import { uploadStatusStore } from "../stores/upload-status";
import { getZipFileStream } from "./fs";
export const listZipItems = async (zipPath: string): Promise<ZipItem[]> => {
const zip = new StreamZip.async({ file: zipPath });
@ -99,51 +98,3 @@ export const markUploadedZipItems = async (
};
export const clearPendingUploads = () => uploadStatusStore.clear();
export const getElectronFilesFromGoogleZip = async (filePath: string) => {
const zip = new StreamZip.async({
file: filePath,
});
const zipName = path.basename(filePath, ".zip");
const entries = await zip.entries();
const files: ElectronFile[] = [];
for (const entry of Object.values(entries)) {
const basename = path.basename(entry.name);
if (entry.isFile && basename.length > 0 && basename[0] !== ".") {
files.push(await getZipEntryAsElectronFile(zipName, zip, entry));
}
}
zip.close();
return files;
};
export async function getZipEntryAsElectronFile(
zipName: string,
zip: StreamZip.StreamZipAsync,
entry: StreamZip.ZipEntry,
): Promise<ElectronFile> {
return {
path: path
.join(zipName, entry.name)
.split(path.sep)
.join(path.posix.sep),
name: path.basename(entry.name),
size: entry.size,
lastModified: entry.time,
stream: async () => {
return await getZipFileStream(zip, entry.name);
},
blob: async () => {
const buffer = await zip.entryData(entry.name);
return new Blob([new Uint8Array(buffer)]);
},
arrayBuffer: async () => {
const buffer = await zip.entryData(entry.name);
return new Uint8Array(buffer);
},
};
}

8
desktop/src/main/services/watch.ts
View file

@ -6,6 +6,7 @@ import { FolderWatch, type CollectionMapping } from "../../types/ipc";
import { fsIsDir } from "../fs";
import log from "../log";
import { watchStore } from "../stores/watch";
import { posixPath } from "../utils-path";
/**
* Create and return a new file system watcher.
@ -46,13 +47,6 @@ const eventData = (path: string): [string, FolderWatch] => {
return [path, watch];
};
/**
* Convert a file system {@link filePath} that uses the local system specific
* path separators into a path that uses POSIX file separators.
*/
const posixPath = (filePath: string) =>
filePath.split(path.sep).join(path.posix.sep);
export const watchGet = (watcher: FSWatcher) => {
const [valid, deleted] = folderWatches().reduce(
([valid, deleted], watch) => {

8
desktop/src/main/utils-path.ts Normal file
View file

@ -0,0 +1,8 @@
import path from "node:path";
/**
* Convert a file system {@link filePath} that uses the local system specific
* path separators into a path that uses POSIX file separators.
*/
export const posixPath = (filePath: string) =>
filePath.split(path.sep).join(path.posix.sep);

2
desktop/src/main/utils-temp.ts
View file

@ -2,7 +2,7 @@ import { app } from "electron/main";
import StreamZip from "node-stream-zip";
import { existsSync } from "node:fs";
import fs from "node:fs/promises";
import path from "path";
import path from "node:path";
import type { ZipItem } from "../types/ipc";
/**

14
desktop/src/preload.ts
View file

@ -63,6 +63,9 @@ const openDirectory = (dirPath: string): Promise<void> =>
const openLogDirectory = (): Promise<void> =>
ipcRenderer.invoke("openLogDirectory");
const selectDirectory = (): Promise<string | undefined> =>
ipcRenderer.invoke("selectDirectory");
const clearStores = () => ipcRenderer.send("clearStores");
const encryptionKey = (): Promise<string | undefined> =>
@ -174,9 +177,6 @@ const faceEmbedding = (input: Float32Array): Promise<Float32Array> =>
// TODO: Deprecated - use dialogs on the renderer process itself
const selectDirectory = (): Promise<string> =>
ipcRenderer.invoke("selectDirectory");
const showUploadFilesDialog = (): Promise<ElectronFile[]> =>
ipcRenderer.invoke("showUploadFilesDialog");
@ -310,6 +310,7 @@ contextBridge.exposeInMainWorld("electron", {
logToDisk,
openDirectory,
openLogDirectory,
selectDirectory,
clearStores,
encryptionKey,
saveEncryptionKey,
@ -348,13 +349,6 @@ contextBridge.exposeInMainWorld("electron", {
detectFaces,
faceEmbedding,
// - File selection
selectDirectory,
showUploadFilesDialog,
showUploadDirsDialog,
showUploadZipDialog,
// - Watch
watch: {

30
web/packages/next/types/ipc.ts
View file

@ -3,8 +3,6 @@
//
// See [Note: types.ts <-> preload.ts <-> ipc.ts]
import type { ElectronFile } from "./file";
/**
* Extra APIs provided by our Node.js layer when our code is running inside our
* desktop (Electron) app.
@ -51,6 +49,18 @@ export interface Electron {
*/
openLogDirectory: () => Promise<void>;
/**
* Ask the user to select a directory on their local file system, and return
* it path.
*
* We don't strictly need IPC for this, we can use a hidden <input> element
* and trigger its click for the same behaviour (as we do for the
* `useFileInput` hook that we use for uploads). However, it's a bit
* cumbersome, and we anyways will need to IPC to get back its full path, so
* it is just convenient to expose this direct method.
*/
selectDirectory: () => Promise<string | undefined>;
/**
* Clear any stored data.
*
@ -122,6 +132,8 @@ export interface Electron {
*/
skipAppUpdate: (version: string) => void;
// - FS
/**
* A subset of file system access APIs.
*
@ -332,20 +344,6 @@ export interface Electron {
*/
faceEmbedding: (input: Float32Array) => Promise<Float32Array>;
// - File selection
// TODO: Deprecated - use dialogs on the renderer process itself
selectDirectory: () => Promise<string>;
showUploadFilesDialog: () => Promise<ElectronFile[]>;
showUploadDirsDialog: () => Promise<ElectronFile[]>;
showUploadZipDialog: () => Promise<{
zipPaths: string[];
files: ElectronFile[];
}>;
// - Watch
/**

23
web/packages/shared/hooks/useFileInput.tsx
View file

@ -1,28 +1,5 @@
import { useCallback, useRef, useState } from "react";
/**
* [Note: File paths when running under Electron]
*
* We have access to the absolute path of the web {@link File} object when we
* are running in the context of our desktop app.
*
* https://www.electronjs.org/docs/latest/api/file-object
*
* This is in contrast to the `webkitRelativePath` that we get when we're
* running in the browser, which is the relative path to the directory that the
* user selected (or just the name of the file if the user selected or
* drag/dropped a single one).
*
* Note that this is a deprecated approach. From Electron docs:
*
* > Warning: The path property that Electron adds to the File interface is
* > deprecated and will be removed in a future Electron release. We recommend
* > you use `webUtils.getPathForFile` instead.
*/
export interface FileWithPath extends File {
readonly path?: string;
}
interface UseFileInputParams {
directory?: boolean;
accept?: string;