beenull/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main' into beta

Browse source
This commit is contained in:
Prateek Sunal 2024-03-25 18:41:14 +05:30
commit 704abf1265
31 changed files with 1160 additions and 921 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
desktop/docs/dependencies.md
View file

@ -52,3 +52,6 @@ Some extra ones specific to the code here are:
* [concurrently](https://github.com/open-cli-tools/concurrently) for spawning
parallel tasks when we do `yarn dev`.
* [shx](https://github.com/shelljs/shx) for providing a portable way to use
Unix commands in scripts. This allows us to use the same commands across
different platforms like Linux and Windows.

3
desktop/package.json
View file

@ -9,7 +9,7 @@
"build": "yarn build-renderer && yarn build-main",
"build-main": "tsc && electron-builder",
"build-main:quick": "tsc && electron-builder --dir --config.compression=store --config.mac.identity=null",
"build-renderer": "cd ../web && yarn install && yarn build:photos && cd ../desktop && rm -f out && ln -sf ../web/apps/photos/out",
"build-renderer": "cd ../web && yarn install && yarn build:photos && cd ../desktop && shx rm -f out && shx ln -sf ../web/apps/photos/out out",
"build:quick": "yarn build-renderer && yarn build-main:quick",
"dev": "concurrently --names 'main,rndr,tscw' \"yarn dev-main\" \"yarn dev-renderer\" \"yarn dev-main-watch\"",
"dev-main": "tsc && electron app/main.js",
@ -47,6 +47,7 @@
"prettier": "^3",
"prettier-plugin-organize-imports": "^3.2",
"prettier-plugin-packagejson": "^2.4",
"shx": "^0.3.4",
"typescript": "^5"
},
"productName": "ente"

44
desktop/src/api/ffmpeg.ts
View file

@ -1,44 +0,0 @@
import { ipcRenderer } from "electron";
import { existsSync } from "fs";
import { writeStream } from "../services/fs";
import { logError } from "../main/log";
import { ElectronFile } from "../types";
export async function runFFmpegCmd(
cmd: string[],
inputFile: File | ElectronFile,
outputFileName: string,
dontTimeout?: boolean,
) {
let inputFilePath = null;
let createdTempInputFile = null;
try {
if (!existsSync(inputFile.path)) {
const tempFilePath = await ipcRenderer.invoke(
"get-temp-file-path",
inputFile.name,
);
await writeStream(tempFilePath, await inputFile.stream());
inputFilePath = tempFilePath;
createdTempInputFile = true;
} else {
inputFilePath = inputFile.path;
}
const outputFileData = await ipcRenderer.invoke(
"run-ffmpeg-cmd",
cmd,
inputFilePath,
outputFileName,
dontTimeout,
);
return new File([outputFileData], outputFileName);
} finally {
if (createdTempInputFile) {
try {
await ipcRenderer.invoke("remove-temp-file", inputFilePath);
} catch (e) {
logError(e, "failed to deleteTempFile");
}
}
}
}

7
desktop/src/api/fs.ts
View file

@ -1,7 +0,0 @@
import { getDirFilePaths, getElectronFile } from "../services/fs";
export async function getDirFiles(dirPath: string) {
const files = await getDirFilePaths(dirPath);
const electronFiles = await Promise.all(files.map(getElectronFile));
return electronFiles;
}

64
desktop/src/api/imageProcessor.ts
View file

@ -1,64 +0,0 @@
import { ipcRenderer } from "electron/renderer";
import { existsSync } from "fs";
import { CustomErrors } from "../constants/errors";
import { writeStream } from "../services/fs";
import { logError } from "../main/log";
import { ElectronFile } from "../types";
import { isPlatform } from "../utils/common/platform";
export async function convertToJPEG(
fileData: Uint8Array,
filename: string,
): Promise<Uint8Array> {
if (isPlatform("windows")) {
throw Error(CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED);
}
const convertedFileData = await ipcRenderer.invoke(
"convert-to-jpeg",
fileData,
filename,
);
return convertedFileData;
}
export async function generateImageThumbnail(
inputFile: File | ElectronFile,
maxDimension: number,
maxSize: number,
): Promise<Uint8Array> {
let inputFilePath = null;
let createdTempInputFile = null;
try {
if (isPlatform("windows")) {
throw Error(
CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED,
);
}
if (!existsSync(inputFile.path)) {
const tempFilePath = await ipcRenderer.invoke(
"get-temp-file-path",
inputFile.name,
);
await writeStream(tempFilePath, await inputFile.stream());
inputFilePath = tempFilePath;
createdTempInputFile = true;
} else {
inputFilePath = inputFile.path;
}
const thumbnail = await ipcRenderer.invoke(
"generate-image-thumbnail",
inputFilePath,
maxDimension,
maxSize,
);
return thumbnail;
} finally {
if (createdTempInputFile) {
try {
await ipcRenderer.invoke("remove-temp-file", inputFilePath);
} catch (e) {
logError(e, "failed to deleteTempFile");
}
}
}
}

14
desktop/src/api/safeStorage.ts
View file

@ -1,13 +1,11 @@
import { ipcRenderer } from "electron";
import { safeStorage } from "electron/main";
import { logError } from "../main/log";
import { safeStorageStore } from "../stores/safeStorage.store";
export async function setEncryptionKey(encryptionKey: string) {
try {
const encryptedKey: Buffer = await ipcRenderer.invoke(
"safeStorage-encrypt",
encryptionKey,
);
const encryptedKey: Buffer =
await safeStorage.encryptString(encryptionKey);
const b64EncryptedKey = Buffer.from(encryptedKey).toString("base64");
safeStorageStore.set("encryptionKey", b64EncryptedKey);
} catch (e) {
@ -20,10 +18,8 @@ export async function getEncryptionKey(): Promise<string> {
try {
const b64EncryptedKey = safeStorageStore.get("encryptionKey");
if (b64EncryptedKey) {
const keyBuffer = new Uint8Array(
Buffer.from(b64EncryptedKey, "base64"),
);
return await ipcRenderer.invoke("safeStorage-decrypt", keyBuffer);
const keyBuffer = Buffer.from(b64EncryptedKey, "base64");
return await safeStorage.decryptString(keyBuffer);
}
} catch (e) {
logError(e, "getEncryptionKey failed");

49
desktop/src/api/upload.ts
View file

@ -1,5 +1,3 @@
import { ipcRenderer } from "electron";
import { logError } from "../main/log";
import {
getElectronFilesFromGoogleZip,
getSavedFilePaths,
@ -36,53 +34,6 @@ export const getPendingUploads = async () => {
};
};
export const showUploadDirsDialog = async () => {
try {
const filePaths: string[] = await ipcRenderer.invoke(
"show-upload-dirs-dialog",
);
const files = await Promise.all(filePaths.map(getElectronFile));
return files;
} catch (e) {
logError(e, "error while selecting folders");
}
};
export const showUploadFilesDialog = async () => {
try {
const filePaths: string[] = await ipcRenderer.invoke(
"show-upload-files-dialog",
);
const files = await Promise.all(filePaths.map(getElectronFile));
return files;
} catch (e) {
logError(e, "error while selecting files");
}
};
export const showUploadZipDialog = async () => {
try {
const filePaths: string[] = await ipcRenderer.invoke(
"show-upload-zip-dialog",
);
let files: ElectronFile[] = [];
for (const filePath of filePaths) {
files = [
...files,
...(await getElectronFilesFromGoogleZip(filePath)),
];
}
return {
zipPaths: filePaths,
files,
};
} catch (e) {
logError(e, "error while selecting zips");
}
};
export {
getElectronFilesFromGoogleZip,
setToUploadCollection,

114
desktop/src/api/watch.ts
View file

@ -1,114 +0,0 @@
import { ipcRenderer } from "electron";
import ElectronLog from "electron-log";
import path from "path";
import { getElectronFile } from "../services/fs";
import { getWatchMappings, setWatchMappings } from "../services/watch";
import { ElectronFile, WatchMapping } from "../types";
import { isMappingPresent } from "../utils/watch";
export async function addWatchMapping(
rootFolderName: string,
folderPath: string,
uploadStrategy: number,
) {
ElectronLog.log(`Adding watch mapping: ${folderPath}`);
const watchMappings = getWatchMappings();
if (isMappingPresent(watchMappings, folderPath)) {
throw new Error(`Watch mapping already exists`);
}
await ipcRenderer.invoke("add-watcher", {
dir: folderPath,
});
watchMappings.push({
rootFolderName,
uploadStrategy,
folderPath,
syncedFiles: [],
ignoredFiles: [],
});
setWatchMappings(watchMappings);
}
export async function removeWatchMapping(folderPath: string) {
let watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw new Error(`Watch mapping does not exist`);
}
await ipcRenderer.invoke("remove-watcher", {
dir: watchMapping.folderPath,
});
watchMappings = watchMappings.filter(
(mapping) => mapping.folderPath !== watchMapping.folderPath,
);
setWatchMappings(watchMappings);
}
export function updateWatchMappingSyncedFiles(
folderPath: string,
files: WatchMapping["syncedFiles"],
): void {
const watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw Error(`Watch mapping not found`);
}
watchMapping.syncedFiles = files;
setWatchMappings(watchMappings);
}
export function updateWatchMappingIgnoredFiles(
folderPath: string,
files: WatchMapping["ignoredFiles"],
): void {
const watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw Error(`Watch mapping not found`);
}
watchMapping.ignoredFiles = files;
setWatchMappings(watchMappings);
}
export function registerWatcherFunctions(
addFile: (file: ElectronFile) => Promise<void>,
removeFile: (path: string) => Promise<void>,
removeFolder: (folderPath: string) => Promise<void>,
) {
ipcRenderer.removeAllListeners("watch-add");
ipcRenderer.removeAllListeners("watch-change");
ipcRenderer.removeAllListeners("watch-unlink-dir");
ipcRenderer.on("watch-add", async (_, filePath: string) => {
filePath = filePath.split(path.sep).join(path.posix.sep);
await addFile(await getElectronFile(filePath));
});
ipcRenderer.on("watch-unlink", async (_, filePath: string) => {
filePath = filePath.split(path.sep).join(path.posix.sep);
await removeFile(filePath);
});
ipcRenderer.on("watch-unlink-dir", async (_, folderPath: string) => {
folderPath = folderPath.split(path.sep).join(path.posix.sep);
await removeFolder(folderPath);
});
}
export { getWatchMappings } from "../services/watch";

14
desktop/src/constants/errors.ts
View file

@ -1,3 +1,17 @@
/**
* [Note: Custom errors across Electron/Renderer boundary]
*
* We need to use the `message` field to disambiguate between errors thrown by
* the main process when invoked from the renderer process. This is because:
*
* > Errors thrown throw `handle` in the main process are not transparent as
* > they are serialized and only the `message` property from the original error
* > is provided to the renderer process.
* >
* > - https://www.electronjs.org/docs/latest/tutorial/ipc
* >
* > Ref: https://github.com/electron/electron/issues/24427
*/
export const CustomErrors = {
WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED:
"Windows native image processing is not supported",

8
desktop/src/main.ts
View file

@ -15,12 +15,11 @@ import { existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { isDev } from "./main/general";
import { attachFSWatchIPCHandlers, attachIPCHandlers } from "./main/ipc";
import { logErrorSentry, setupLogging } from "./main/log";
import { initWatcher } from "./services/chokidar";
import { addAllowOriginHeader } from "./utils/cors";
import { createWindow } from "./utils/createWindow";
import setupIpcComs from "./utils/ipcComms";
import {
handleDockIconHideOnAutoLaunch,
handleDownloads,
@ -167,11 +166,12 @@ const main = () => {
app.on("ready", async () => {
logSystemInfo();
mainWindow = await createWindow();
const tray = setupTrayItem(mainWindow);
const watcher = initWatcher(mainWindow);
setupTrayItem(mainWindow);
setupMacWindowOnDockIconClick();
setupMainMenu(mainWindow);
setupIpcComs(tray, mainWindow, watcher);
attachIPCHandlers();
attachFSWatchIPCHandlers(watcher);
await handleUpdates(mainWindow);
handleDownloads(mainWindow);
handleExternalLinks(mainWindow);

54
desktop/src/main/dialogs.ts Normal file
View file

@ -0,0 +1,54 @@
import { dialog } from "electron/main";
import * as path from "node:path";
import { getDirFilePaths, getElectronFile } from "../services/fs";
import { getElectronFilesFromGoogleZip } from "../services/upload";
import type { ElectronFile } from "../types";
export const selectDirectory = async () => {
const result = await dialog.showOpenDialog({
properties: ["openDirectory"],
});
if (result.filePaths && result.filePaths.length > 0) {
return result.filePaths[0]?.split(path.sep)?.join(path.posix.sep);
}
};
export const showUploadFilesDialog = async () => {
const selectedFiles = await dialog.showOpenDialog({
properties: ["openFile", "multiSelections"],
});
const filePaths = selectedFiles.filePaths;
return await Promise.all(filePaths.map(getElectronFile));
};
export const showUploadDirsDialog = async () => {
const dir = await dialog.showOpenDialog({
properties: ["openDirectory", "multiSelections"],
});
let filePaths: string[] = [];
for (const dirPath of dir.filePaths) {
filePaths = [...filePaths, ...(await getDirFilePaths(dirPath))];
}
return await Promise.all(filePaths.map(getElectronFile));
};
export const showUploadZipDialog = async () => {
const selectedFiles = await dialog.showOpenDialog({
properties: ["openFile", "multiSelections"],
filters: [{ name: "Zip File", extensions: ["zip"] }],
});
const filePaths = selectedFiles.filePaths;
let files: ElectronFile[] = [];
for (const filePath of filePaths) {
files = [...files, ...(await getElectronFilesFromGoogleZip(filePath))];
}
return {
zipPaths: filePaths,
files,
};
};

142
desktop/src/main/fs.ts
View file

@ -1,12 +1,152 @@
/**
* @file file system related functions exposed over the context bridge.
*/
import { existsSync } from "node:fs";
import { createWriteStream, existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { Readable } from "node:stream";
import { logError } from "./log";
export const fsExists = (path: string) => existsSync(path);
/**
* Write a (web) ReadableStream to a file at the given {@link filePath}.
*
* The returned promise resolves when the write completes.
*
* @param filePath The local filesystem path where the file should be written.
* @param readableStream A [web
* ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream)
*/
export const writeStream = (filePath: string, readableStream: ReadableStream) =>
writeNodeStream(filePath, convertWebReadableStreamToNode(readableStream));
/**
* Convert a Web ReadableStream into a Node.js ReadableStream
*
* This can be used to, for example, write a ReadableStream obtained via
* `net.fetch` into a file using the Node.js `fs` APIs
*/
const convertWebReadableStreamToNode = (readableStream: ReadableStream) => {
const reader = readableStream.getReader();
const rs = new Readable();
rs._read = async () => {
try {
const result = await reader.read();
if (!result.done) {
rs.push(Buffer.from(result.value));
} else {
rs.push(null);
return;
}
} catch (e) {
rs.emit("error", e);
}
};
return rs;
};
const writeNodeStream = async (
filePath: string,
fileStream: NodeJS.ReadableStream,
) => {
const writeable = createWriteStream(filePath);
fileStream.on("error", (error) => {
writeable.destroy(error); // Close the writable stream with an error
});
fileStream.pipe(writeable);
await new Promise((resolve, reject) => {
writeable.on("finish", resolve);
writeable.on("error", async (e: unknown) => {
if (existsSync(filePath)) {
await fs.unlink(filePath);
}
reject(e);
});
});
};
/* TODO: Audit below this */
export const checkExistsAndCreateDir = (dirPath: string) =>
fs.mkdir(dirPath, { recursive: true });
export const saveStreamToDisk = writeStream;
export const saveFileToDisk = (path: string, contents: string) =>
fs.writeFile(path, contents);
export const readTextFile = async (filePath: string) =>
fs.readFile(filePath, "utf-8");
export const moveFile = async (sourcePath: string, destinationPath: string) => {
if (!existsSync(sourcePath)) {
throw new Error("File does not exist");
}
if (existsSync(destinationPath)) {
throw new Error("Destination file already exists");
}
// check if destination folder exists
const destinationFolder = path.dirname(destinationPath);
await fs.mkdir(destinationFolder, { recursive: true });
await fs.rename(sourcePath, destinationPath);
};
export const isFolder = async (dirPath: string) => {
try {
const stats = await fs.stat(dirPath);
return stats.isDirectory();
} catch (e) {
let err = e;
// if code is defined, it's an error from fs.stat
if (typeof e.code !== "undefined") {
// ENOENT means the file does not exist
if (e.code === "ENOENT") {
return false;
}
err = Error(`fs error code: ${e.code}`);
}
logError(err, "isFolder failed");
return false;
}
};
export const deleteFolder = async (folderPath: string) => {
if (!existsSync(folderPath)) {
return;
}
const stat = await fs.stat(folderPath);
if (!stat.isDirectory()) {
throw new Error("Path is not a folder");
}
// check if folder is empty
const files = await fs.readdir(folderPath);
if (files.length > 0) {
throw new Error("Folder is not empty");
}
await fs.rmdir(folderPath);
};
export const rename = async (oldPath: string, newPath: string) => {
if (!existsSync(oldPath)) {
throw new Error("Path does not exist");
}
await fs.rename(oldPath, newPath);
};
export const deleteFile = async (filePath: string) => {
if (!existsSync(filePath)) {
return;
}
const stat = await fs.stat(filePath);
if (!stat.isFile()) {
throw new Error("Path is not a file");
}
return fs.rm(filePath);
};

223
desktop/src/main/ipc.ts
View file

@ -6,18 +6,70 @@
* context of the main process, and can import other files from `src/`.
*/
import type { FSWatcher } from "chokidar";
import { ipcMain } from "electron/main";
import { clearElectronStore } from "../api/electronStore";
import { getEncryptionKey, setEncryptionKey } from "../api/safeStorage";
import {
getElectronFilesFromGoogleZip,
getPendingUploads,
setToUploadCollection,
setToUploadFiles,
} from "../api/upload";
import {
appVersion,
muteUpdateNotification,
skipAppUpdate,
updateAndRestart,
} from "../services/appUpdater";
import { checkExistsAndCreateDir, fsExists } from "./fs";
import {
computeImageEmbedding,
computeTextEmbedding,
} from "../services/clipService";
import { runFFmpegCmd } from "../services/ffmpeg";
import { getDirFiles } from "../services/fs";
import {
convertToJPEG,
generateImageThumbnail,
} from "../services/imageProcessor";
import {
addWatchMapping,
getWatchMappings,
removeWatchMapping,
updateWatchMappingIgnoredFiles,
updateWatchMappingSyncedFiles,
} from "../services/watch";
import type {
ElectronFile,
FILE_PATH_TYPE,
Model,
WatchMapping,
} from "../types";
import {
selectDirectory,
showUploadDirsDialog,
showUploadFilesDialog,
showUploadZipDialog,
} from "./dialogs";
import {
checkExistsAndCreateDir,
deleteFile,
deleteFolder,
fsExists,
isFolder,
moveFile,
readTextFile,
rename,
saveFileToDisk,
saveStreamToDisk,
} from "./fs";
import { openDirectory, openLogDirectory } from "./general";
import { logToDisk } from "./log";
/**
* Listen for IPC events sent/invoked by the renderer process, and route them to
* their correct handlers.
*/
export const attachIPCHandlers = () => {
// Notes:
//
@ -46,24 +98,169 @@ export const attachIPCHandlers = () => {
// See: [Note: Catching exception during .send/.on]
ipcMain.on("logToDisk", (_, message) => logToDisk(message));
ipcMain.on("clear-electron-store", (_) => {
clearElectronStore();
});
ipcMain.handle("setEncryptionKey", (_, encryptionKey) =>
setEncryptionKey(encryptionKey),
);
ipcMain.handle("getEncryptionKey", (_) => getEncryptionKey());
// - App update
ipcMain.on("update-and-restart", (_) => updateAndRestart());
ipcMain.on("skip-app-update", (_, version) => skipAppUpdate(version));
ipcMain.on("mute-update-notification", (_, version) =>
muteUpdateNotification(version),
);
// - Conversion
ipcMain.handle("convertToJPEG", (_, fileData, filename) =>
convertToJPEG(fileData, filename),
);
ipcMain.handle(
"generateImageThumbnail",
(_, inputFile, maxDimension, maxSize) =>
generateImageThumbnail(inputFile, maxDimension, maxSize),
);
ipcMain.handle(
"runFFmpegCmd",
(
_,
cmd: string[],
inputFile: File | ElectronFile,
outputFileName: string,
dontTimeout?: boolean,
) => runFFmpegCmd(cmd, inputFile, outputFileName, dontTimeout),
);
// - ML
ipcMain.handle(
"computeImageEmbedding",
(_, model: Model, imageData: Uint8Array) =>
computeImageEmbedding(model, imageData),
);
ipcMain.handle("computeTextEmbedding", (_, model: Model, text: string) =>
computeTextEmbedding(model, text),
);
// - File selection
ipcMain.handle("selectDirectory", (_) => selectDirectory());
ipcMain.handle("showUploadFilesDialog", (_) => showUploadFilesDialog());
ipcMain.handle("showUploadDirsDialog", (_) => showUploadDirsDialog());
ipcMain.handle("showUploadZipDialog", (_) => showUploadZipDialog());
// - FS
ipcMain.handle("fsExists", (_, path) => fsExists(path));
// - FS Legacy
ipcMain.handle("checkExistsAndCreateDir", (_, dirPath) =>
checkExistsAndCreateDir(dirPath),
);
ipcMain.on("clear-electron-store", (_) => {
clearElectronStore();
});
ipcMain.handle(
"saveStreamToDisk",
(_, path: string, fileStream: ReadableStream<any>) =>
saveStreamToDisk(path, fileStream),
);
ipcMain.on("update-and-restart", (_) => {
updateAndRestart();
});
ipcMain.on("skip-app-update", (_, version) => {
skipAppUpdate(version);
});
ipcMain.handle("saveFileToDisk", (_, path: string, file: any) =>
saveFileToDisk(path, file),
);
ipcMain.on("mute-update-notification", (_, version) => {
muteUpdateNotification(version);
});
ipcMain.handle("readTextFile", (_, path: string) => readTextFile(path));
ipcMain.handle("isFolder", (_, dirPath: string) => isFolder(dirPath));
ipcMain.handle("moveFile", (_, oldPath: string, newPath: string) =>
moveFile(oldPath, newPath),
);
ipcMain.handle("deleteFolder", (_, path: string) => deleteFolder(path));
ipcMain.handle("deleteFile", (_, path: string) => deleteFile(path));
ipcMain.handle("rename", (_, oldPath: string, newPath: string) =>
rename(oldPath, newPath),
);
// - Upload
ipcMain.handle("getPendingUploads", (_) => getPendingUploads());
ipcMain.handle(
"setToUploadFiles",
(_, type: FILE_PATH_TYPE, filePaths: string[]) =>
setToUploadFiles(type, filePaths),
);
ipcMain.handle("getElectronFilesFromGoogleZip", (_, filePath: string) =>
getElectronFilesFromGoogleZip(filePath),
);
ipcMain.handle("setToUploadCollection", (_, collectionName: string) =>
setToUploadCollection(collectionName),
);
ipcMain.handle("getDirFiles", (_, dirPath: string) => getDirFiles(dirPath));
};
/**
* Sibling of {@link attachIPCHandlers} that attaches handlers specific to the
* watch folder functionality.
*
* It gets passed a {@link FSWatcher} instance which it can then forward to the
* actual handlers.
*/
export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => {
// - Watch
ipcMain.handle(
"addWatchMapping",
(
_,
collectionName: string,
folderPath: string,
uploadStrategy: number,
) =>
addWatchMapping(
watcher,
collectionName,
folderPath,
uploadStrategy,
),
);
ipcMain.handle("removeWatchMapping", (_, folderPath: string) =>
removeWatchMapping(watcher, folderPath),
);
ipcMain.handle("getWatchMappings", (_) => getWatchMappings());
ipcMain.handle(
"updateWatchMappingSyncedFiles",
(_, folderPath: string, files: WatchMapping["syncedFiles"]) =>
updateWatchMappingSyncedFiles(folderPath, files),
);
ipcMain.handle(
"updateWatchMappingIgnoredFiles",
(_, folderPath: string, files: WatchMapping["ignoredFiles"]) =>
updateWatchMappingIgnoredFiles(folderPath, files),
);
};

550
desktop/src/preload.ts
View file

@ -27,34 +27,11 @@
*/
import { contextBridge, ipcRenderer } from "electron";
import { createWriteStream, existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import { Readable } from "node:stream";
import path from "path";
import { runFFmpegCmd } from "./api/ffmpeg";
import { getDirFiles } from "./api/fs";
import { convertToJPEG, generateImageThumbnail } from "./api/imageProcessor";
import { getEncryptionKey, setEncryptionKey } from "./api/safeStorage";
import {
getElectronFilesFromGoogleZip,
getPendingUploads,
setToUploadCollection,
setToUploadFiles,
showUploadDirsDialog,
showUploadFilesDialog,
showUploadZipDialog,
} from "./api/upload";
import {
addWatchMapping,
getWatchMappings,
registerWatcherFunctions,
removeWatchMapping,
updateWatchMappingIgnoredFiles,
updateWatchMappingSyncedFiles,
} from "./api/watch";
import { logErrorSentry, setupLogging } from "./main/log";
import type { ElectronFile } from "./types";
setupLogging();
// TODO (MR): Uncomment and FIXME once preload is getting loaded.
// import { setupLogging } from "./main/log";
// setupLogging();
// - General
@ -93,8 +70,24 @@ const fsExists = (path: string): Promise<boolean> =>
// - AUDIT below this
const checkExistsAndCreateDir = (dirPath: string): Promise<void> =>
ipcRenderer.invoke("checkExistsAndCreateDir", dirPath);
const registerForegroundEventListener = (onForeground: () => void) => {
ipcRenderer.removeAllListeners("app-in-foreground");
ipcRenderer.on("app-in-foreground", () => {
onForeground();
});
};
const clearElectronStore = () => {
ipcRenderer.send("clear-electron-store");
};
const setEncryptionKey = (encryptionKey: string): Promise<void> =>
ipcRenderer.invoke("setEncryptionKey", encryptionKey);
const getEncryptionKey = (): Promise<string> =>
ipcRenderer.invoke("getEncryptionKey");
// - App update
/* preload: duplicated */
interface AppUpdateInfo {
@ -111,19 +104,6 @@ const registerUpdateEventListener = (
});
};
const registerForegroundEventListener = (onForeground: () => void) => {
ipcRenderer.removeAllListeners("app-in-foreground");
ipcRenderer.on("app-in-foreground", () => {
onForeground();
});
};
const clearElectronStore = () => {
ipcRenderer.send("clear-electron-store");
};
// - App update
const updateAndRestart = () => {
ipcRenderer.send("update-and-restart");
};
@ -136,278 +116,203 @@ const muteUpdateNotification = (version: string) => {
ipcRenderer.send("mute-update-notification", version);
};
// - Conversion
// - FIXME below this
const convertToJPEG = (
fileData: Uint8Array,
filename: string,
): Promise<Uint8Array> =>
ipcRenderer.invoke("convertToJPEG", fileData, filename);
/* preload: duplicated logError */
const logError = (error: Error, message: string, info?: any) => {
logErrorSentry(error, message, info);
};
const generateImageThumbnail = (
inputFile: File | ElectronFile,
maxDimension: number,
maxSize: number,
): Promise<Uint8Array> =>
ipcRenderer.invoke(
"generateImageThumbnail",
inputFile,
maxDimension,
maxSize,
);
/* preload: duplicated writeStream */
/**
* Write a (web) ReadableStream to a file at the given {@link filePath}.
*
* The returned promise resolves when the write completes.
*
* @param filePath The local filesystem path where the file should be written.
* @param readableStream A [web
* ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream)
*/
const writeStream = (filePath: string, readableStream: ReadableStream) =>
writeNodeStream(filePath, convertWebReadableStreamToNode(readableStream));
/**
* Convert a Web ReadableStream into a Node.js ReadableStream
*
* This can be used to, for example, write a ReadableStream obtained via
* `net.fetch` into a file using the Node.js `fs` APIs
*/
const convertWebReadableStreamToNode = (readableStream: ReadableStream) => {
const reader = readableStream.getReader();
const rs = new Readable();
rs._read = async () => {
try {
const result = await reader.read();
if (!result.done) {
rs.push(Buffer.from(result.value));
} else {
rs.push(null);
return;
}
} catch (e) {
rs.emit("error", e);
}
};
return rs;
};
const writeNodeStream = async (
filePath: string,
fileStream: NodeJS.ReadableStream,
) => {
const writeable = createWriteStream(filePath);
fileStream.on("error", (error) => {
writeable.destroy(error); // Close the writable stream with an error
});
fileStream.pipe(writeable);
await new Promise((resolve, reject) => {
writeable.on("finish", resolve);
writeable.on("error", async (e: unknown) => {
if (existsSync(filePath)) {
await fs.unlink(filePath);
}
reject(e);
});
});
};
// - Export
const saveStreamToDisk = writeStream;
const saveFileToDisk = (path: string, contents: string) =>
fs.writeFile(path, contents);
// -
async function readTextFile(filePath: string) {
if (!existsSync(filePath)) {
throw new Error("File does not exist");
}
return await fs.readFile(filePath, "utf-8");
}
async function moveFile(
sourcePath: string,
destinationPath: string,
): Promise<void> {
if (!existsSync(sourcePath)) {
throw new Error("File does not exist");
}
if (existsSync(destinationPath)) {
throw new Error("Destination file already exists");
}
// check if destination folder exists
const destinationFolder = path.dirname(destinationPath);
await fs.mkdir(destinationFolder, { recursive: true });
await fs.rename(sourcePath, destinationPath);
}
export async function isFolder(dirPath: string) {
try {
const stats = await fs.stat(dirPath);
return stats.isDirectory();
} catch (e) {
let err = e;
// if code is defined, it's an error from fs.stat
if (typeof e.code !== "undefined") {
// ENOENT means the file does not exist
if (e.code === "ENOENT") {
return false;
}
err = Error(`fs error code: ${e.code}`);
}
logError(err, "isFolder failed");
return false;
}
}
async function deleteFolder(folderPath: string): Promise<void> {
if (!existsSync(folderPath)) {
return;
}
const stat = await fs.stat(folderPath);
if (!stat.isDirectory()) {
throw new Error("Path is not a folder");
}
// check if folder is empty
const files = await fs.readdir(folderPath);
if (files.length > 0) {
throw new Error("Folder is not empty");
}
await fs.rmdir(folderPath);
}
async function rename(oldPath: string, newPath: string) {
if (!existsSync(oldPath)) {
throw new Error("Path does not exist");
}
await fs.rename(oldPath, newPath);
}
const deleteFile = async (filePath: string) => {
if (!existsSync(filePath)) {
return;
}
const stat = await fs.stat(filePath);
if (!stat.isFile()) {
throw new Error("Path is not a file");
}
return fs.rm(filePath);
};
const runFFmpegCmd = (
cmd: string[],
inputFile: File | ElectronFile,
outputFileName: string,
dontTimeout?: boolean,
): Promise<File> =>
ipcRenderer.invoke(
"runFFmpegCmd",
cmd,
inputFile,
outputFileName,
dontTimeout,
);
// - ML
/* preload: duplicated Model */
export enum Model {
enum Model {
GGML_CLIP = "ggml-clip",
ONNX_CLIP = "onnx-clip",
}
const computeImageEmbedding = async (
const computeImageEmbedding = (
model: Model,
imageData: Uint8Array,
): Promise<Float32Array> => {
let tempInputFilePath = null;
try {
tempInputFilePath = await ipcRenderer.invoke("get-temp-file-path", "");
const imageStream = new Response(imageData.buffer).body;
await writeStream(tempInputFilePath, imageStream);
const embedding = await ipcRenderer.invoke(
"compute-image-embedding",
model,
tempInputFilePath,
);
return embedding;
} catch (err) {
if (isExecError(err)) {
const parsedExecError = parseExecError(err);
throw Error(parsedExecError);
} else {
throw err;
}
} finally {
if (tempInputFilePath) {
await ipcRenderer.invoke("remove-temp-file", tempInputFilePath);
}
}
};
): Promise<Float32Array> =>
ipcRenderer.invoke("computeImageEmbedding", model, imageData);
export async function computeTextEmbedding(
const computeTextEmbedding = (
model: Model,
text: string,
): Promise<Float32Array> {
try {
const embedding = await ipcRenderer.invoke(
"compute-text-embedding",
model,
text,
);
return embedding;
} catch (err) {
if (isExecError(err)) {
const parsedExecError = parseExecError(err);
throw Error(parsedExecError);
} else {
throw err;
}
}
): Promise<Float32Array> =>
ipcRenderer.invoke("computeTextEmbedding", model, text);
// - File selection
// TODO: Deprecated - use dialogs on the renderer process itself
const selectDirectory = (): Promise<string> =>
ipcRenderer.invoke("selectDirectory");
const showUploadFilesDialog = (): Promise<ElectronFile[]> =>
ipcRenderer.invoke("showUploadFilesDialog");
const showUploadDirsDialog = (): Promise<ElectronFile[]> =>
ipcRenderer.invoke("showUploadDirsDialog");
const showUploadZipDialog = (): Promise<{
zipPaths: string[];
files: ElectronFile[];
}> => ipcRenderer.invoke("showUploadZipDialog");
// - Watch
const registerWatcherFunctions = (
addFile: (file: ElectronFile) => Promise<void>,
removeFile: (path: string) => Promise<void>,
removeFolder: (folderPath: string) => Promise<void>,
) => {
ipcRenderer.removeAllListeners("watch-add");
ipcRenderer.removeAllListeners("watch-unlink");
ipcRenderer.removeAllListeners("watch-unlink-dir");
ipcRenderer.on("watch-add", (_, file: ElectronFile) => addFile(file));
ipcRenderer.on("watch-unlink", (_, filePath: string) =>
removeFile(filePath),
);
ipcRenderer.on("watch-unlink-dir", (_, folderPath: string) =>
removeFolder(folderPath),
);
};
const addWatchMapping = (
collectionName: string,
folderPath: string,
uploadStrategy: number,
): Promise<void> =>
ipcRenderer.invoke(
"addWatchMapping",
collectionName,
folderPath,
uploadStrategy,
);
const removeWatchMapping = (folderPath: string): Promise<void> =>
ipcRenderer.invoke("removeWatchMapping", folderPath);
/* preload: duplicated WatchMappingSyncedFile */
interface WatchMappingSyncedFile {
path: string;
uploadedFileID: number;
collectionID: number;
}
// -
/* preload: duplicated WatchMapping */
interface WatchMapping {
rootFolderName: string;
uploadStrategy: number;
folderPath: string;
syncedFiles: WatchMappingSyncedFile[];
ignoredFiles: string[];
}
/**
* [Note: Custom errors across Electron/Renderer boundary]
*
* We need to use the `message` field to disambiguate between errors thrown by
* the main process when invoked from the renderer process. This is because:
*
* > Errors thrown throw `handle` in the main process are not transparent as
* > they are serialized and only the `message` property from the original error
* > is provided to the renderer process.
* >
* > - https://www.electronjs.org/docs/latest/tutorial/ipc
* >
* > Ref: https://github.com/electron/electron/issues/24427
*/
/* preload: duplicated CustomErrors */
const CustomErrorsP = {
WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED:
"Windows native image processing is not supported",
INVALID_OS: (os: string) => `Invalid OS - ${os}`,
WAIT_TIME_EXCEEDED: "Wait time exceeded",
UNSUPPORTED_PLATFORM: (platform: string, arch: string) =>
`Unsupported platform - ${platform} ${arch}`,
MODEL_DOWNLOAD_PENDING:
"Model download pending, skipping clip search request",
INVALID_FILE_PATH: "Invalid file path",
INVALID_CLIP_MODEL: (model: string) => `Invalid Clip model - ${model}`,
};
const getWatchMappings = (): Promise<WatchMapping[]> =>
ipcRenderer.invoke("getWatchMappings");
const isExecError = (err: any) => {
return err.message.includes("Command failed:");
};
const updateWatchMappingSyncedFiles = (
folderPath: string,
files: WatchMapping["syncedFiles"],
): Promise<void> =>
ipcRenderer.invoke("updateWatchMappingSyncedFiles", folderPath, files);
const parseExecError = (err: any) => {
const errMessage = err.message;
if (errMessage.includes("Bad CPU type in executable")) {
return CustomErrorsP.UNSUPPORTED_PLATFORM(
process.platform,
process.arch,
);
} else {
return errMessage;
}
};
const updateWatchMappingIgnoredFiles = (
folderPath: string,
files: WatchMapping["ignoredFiles"],
): Promise<void> =>
ipcRenderer.invoke("updateWatchMappingIgnoredFiles", folderPath, files);
// - General
// - FS Legacy
const selectDirectory = async (): Promise<string> => {
try {
return await ipcRenderer.invoke("select-dir");
} catch (e) {
logError(e, "error while selecting root directory");
}
};
const checkExistsAndCreateDir = (dirPath: string): Promise<void> =>
ipcRenderer.invoke("checkExistsAndCreateDir", dirPath);
// -
const saveStreamToDisk = (
path: string,
fileStream: ReadableStream<any>,
): Promise<void> => ipcRenderer.invoke("saveStreamToDisk", path, fileStream);
const saveFileToDisk = (path: string, file: any): Promise<void> =>
ipcRenderer.invoke("saveFileToDisk", path, file);
const readTextFile = (path: string): Promise<string> =>
ipcRenderer.invoke("readTextFile", path);
const isFolder = (dirPath: string): Promise<boolean> =>
ipcRenderer.invoke("isFolder", dirPath);
const moveFile = (oldPath: string, newPath: string): Promise<void> =>
ipcRenderer.invoke("moveFile", oldPath, newPath);
const deleteFolder = (path: string): Promise<void> =>
ipcRenderer.invoke("deleteFolder", path);
const deleteFile = (path: string): Promise<void> =>
ipcRenderer.invoke("deleteFile", path);
const rename = (oldPath: string, newPath: string): Promise<void> =>
ipcRenderer.invoke("rename", oldPath, newPath);
// - Upload
const getPendingUploads = (): Promise<{
files: ElectronFile[];
collectionName: string;
type: string;
}> => ipcRenderer.invoke("getPendingUploads");
/* preload: duplicated FILE_PATH_TYPE */
enum FILE_PATH_TYPE {
FILES = "files",
ZIPS = "zips",
}
const setToUploadFiles = (
type: FILE_PATH_TYPE,
filePaths: string[],
): Promise<void> => ipcRenderer.invoke("setToUploadFiles", type, filePaths);
const getElectronFilesFromGoogleZip = (
filePath: string,
): Promise<ElectronFile[]> =>
ipcRenderer.invoke("getElectronFilesFromGoogleZip", filePath);
const setToUploadCollection = (collectionName: string): Promise<void> =>
ipcRenderer.invoke("setToUploadCollection", collectionName);
const getDirFiles = (dirPath: string): Promise<ElectronFile[]> =>
ipcRenderer.invoke("getDirFiles", dirPath);
// These objects exposed here will become available to the JS code in our
// renderer (the web/ code) as `window.ElectronAPIs.*`
@ -442,13 +347,15 @@ const selectDirectory = async (): Promise<string> => {
// The copy itself is relatively fast, but the problem with transfering large
// amounts of data is potentially running out of memory during the copy.
contextBridge.exposeInMainWorld("ElectronAPIs", {
// General
// - General
appVersion,
openDirectory,
registerForegroundEventListener,
clearElectronStore,
getEncryptionKey,
setEncryptionKey,
// Logging
// - Logging
openLogDirectory,
logToDisk,
@ -458,6 +365,29 @@ contextBridge.exposeInMainWorld("ElectronAPIs", {
muteUpdateNotification,
registerUpdateEventListener,
// - Conversion
convertToJPEG,
generateImageThumbnail,
runFFmpegCmd,
// - ML
computeImageEmbedding,
computeTextEmbedding,
// - File selection
selectDirectory,
showUploadFilesDialog,
showUploadDirsDialog,
showUploadZipDialog,
// - Watch
registerWatcherFunctions,
addWatchMapping,
removeWatchMapping,
getWatchMappings,
updateWatchMappingSyncedFiles,
updateWatchMappingIgnoredFiles,
// - FS
fs: {
exists: fsExists,
@ -466,40 +396,20 @@ contextBridge.exposeInMainWorld("ElectronAPIs", {
// - FS legacy
// TODO: Move these into fs + document + rename if needed
checkExistsAndCreateDir,
// - Export
saveStreamToDisk,
saveFileToDisk,
selectDirectory,
readTextFile,
showUploadFilesDialog,
showUploadDirsDialog,
getPendingUploads,
setToUploadFiles,
showUploadZipDialog,
getElectronFilesFromGoogleZip,
setToUploadCollection,
getEncryptionKey,
setEncryptionKey,
getDirFiles,
getWatchMappings,
addWatchMapping,
removeWatchMapping,
registerWatcherFunctions,
isFolder,
updateWatchMappingSyncedFiles,
updateWatchMappingIgnoredFiles,
convertToJPEG,
runFFmpegCmd,
generateImageThumbnail,
moveFile,
deleteFolder,
rename,
deleteFile,
rename,
// - ML
computeImageEmbedding,
computeTextEmbedding,
// - Upload
getPendingUploads,
setToUploadFiles,
getElectronFilesFromGoogleZip,
setToUploadCollection,
getDirFiles,
});

28
desktop/src/services/chokidar.ts
View file

@ -1,7 +1,16 @@
import chokidar from "chokidar";
import { BrowserWindow } from "electron";
import { getWatchMappings } from "../api/watch";
import * as path from "path";
import { logError } from "../main/log";
import { getWatchMappings } from "../services/watch";
import { getElectronFile } from "./fs";
/**
* Convert a file system {@link filePath} that uses the local system specific
* path separators into a path that uses POSIX file separators.
*/
const normalizeToPOSIX = (filePath: string) =>
filePath.split(path.sep).join(path.posix.sep);
export function initWatcher(mainWindow: BrowserWindow) {
const mappings = getWatchMappings();
@ -13,17 +22,20 @@ export function initWatcher(mainWindow: BrowserWindow) {
awaitWriteFinish: true,
});
watcher
.on("add", (path) => {
mainWindow.webContents.send("watch-add", path);
})
.on("change", (path) => {
mainWindow.webContents.send("watch-change", path);
.on("add", async (path) => {
mainWindow.webContents.send(
"watch-add",
await getElectronFile(normalizeToPOSIX(path)),
);
})
.on("unlink", (path) => {
mainWindow.webContents.send("watch-unlink", path);
mainWindow.webContents.send("watch-unlink", normalizeToPOSIX(path));
})
.on("unlinkDir", (path) => {
mainWindow.webContents.send("watch-unlink-dir", path);
mainWindow.webContents.send(
"watch-unlink-dir",
normalizeToPOSIX(path),
);
})
.on("error", (error) => {
logError(error, "error while watching files");

71
desktop/src/services/clipService.ts
View file

@ -5,12 +5,14 @@ import * as fs from "node:fs/promises";
import * as path from "node:path";
import util from "util";
import { CustomErrors } from "../constants/errors";
import { writeStream } from "../main/fs";
import { isDev } from "../main/general";
import { logErrorSentry } from "../main/log";
import { Model } from "../types";
import Tokenizer from "../utils/clip-bpe-ts/mod";
import { isDev } from "../main/general";
import { getPlatform } from "../utils/common/platform";
import { writeStream } from "./fs";
import { logErrorSentry } from "../main/log";
import { generateTempFilePath } from "../utils/temp";
import { deleteTempFile } from "./ffmpeg";
const shellescape = require("any-shell-escape");
const execAsync = util.promisify(require("child_process").exec);
const jpeg = require("jpeg-js");
@ -198,7 +200,51 @@ function getTokenizer() {
return tokenizer;
}
export async function computeImageEmbedding(
export const computeImageEmbedding = async (
model: Model,
imageData: Uint8Array,
): Promise<Float32Array> => {
let tempInputFilePath = null;
try {
tempInputFilePath = await generateTempFilePath("");
const imageStream = new Response(imageData.buffer).body;
await writeStream(tempInputFilePath, imageStream);
const embedding = await computeImageEmbedding_(
model,
tempInputFilePath,
);
return embedding;
} catch (err) {
if (isExecError(err)) {
const parsedExecError = parseExecError(err);
throw Error(parsedExecError);
} else {
throw err;
}
} finally {
if (tempInputFilePath) {
await deleteTempFile(tempInputFilePath);
}
}
};
const isExecError = (err: any) => {
return err.message.includes("Command failed:");
};
const parseExecError = (err: any) => {
const errMessage = err.message;
if (errMessage.includes("Bad CPU type in executable")) {
return CustomErrors.UNSUPPORTED_PLATFORM(
process.platform,
process.arch,
);
} else {
return errMessage;
}
};
async function computeImageEmbedding_(
model: Model,
inputFilePath: string,
): Promise<Float32Array> {
@ -278,6 +324,23 @@ export async function computeONNXImageEmbedding(
export async function computeTextEmbedding(
model: Model,
text: string,
): Promise<Float32Array> {
try {
const embedding = computeTextEmbedding_(model, text);
return embedding;
} catch (err) {
if (isExecError(err)) {
const parsedExecError = parseExecError(err);
throw Error(parsedExecError);
} else {
throw err;
}
}
}
async function computeTextEmbedding_(
model: Model,
text: string,
): Promise<Float32Array> {
if (model === Model.GGML_CLIP) {
return await computeGGMLTextEmbedding(text);

40
desktop/src/services/ffmpeg.ts
View file

@ -4,8 +4,11 @@ import { existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import util from "util";
import { CustomErrors } from "../constants/errors";
import { writeStream } from "../main/fs";
import { logError, logErrorSentry } from "../main/log";
import { ElectronFile } from "../types";
import { generateTempFilePath, getTempDirPath } from "../utils/temp";
import { logErrorSentry } from "../main/log";
const shellescape = require("any-shell-escape");
const execAsync = util.promisify(require("child_process").exec);
@ -42,6 +45,41 @@ const OUTPUT_PATH_PLACEHOLDER = "OUTPUT";
* I'm not sure if our code is supposed to be able to use it, and how.
*/
export async function runFFmpegCmd(
cmd: string[],
inputFile: File | ElectronFile,
outputFileName: string,
dontTimeout?: boolean,
) {
let inputFilePath = null;
let createdTempInputFile = null;
try {
if (!existsSync(inputFile.path)) {
const tempFilePath = await generateTempFilePath(inputFile.name);
await writeStream(tempFilePath, await inputFile.stream());
inputFilePath = tempFilePath;
createdTempInputFile = true;
} else {
inputFilePath = inputFile.path;
}
const outputFileData = await runFFmpegCmd_(
cmd,
inputFilePath,
outputFileName,
dontTimeout,
);
return new File([outputFileData], outputFileName);
} finally {
if (createdTempInputFile) {
try {
await deleteTempFile(inputFilePath);
} catch (e) {
logError(e, "failed to deleteTempFile");
}
}
}
}
export async function runFFmpegCmd_(
cmd: string[],
inputFilePath: string,
outputFileName: string,

66
desktop/src/services/fs.ts
View file

@ -1,13 +1,18 @@
import StreamZip from "node-stream-zip";
import { createWriteStream, existsSync } from "node:fs";
import { existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { Readable } from "stream";
import { ElectronFile } from "../types";
import { logError } from "../main/log";
import { ElectronFile } from "../types";
const FILE_STREAM_CHUNK_SIZE: number = 4 * 1024 * 1024;
export async function getDirFiles(dirPath: string) {
const files = await getDirFilePaths(dirPath);
const electronFiles = await Promise.all(files.map(getElectronFile));
return electronFiles;
}
// https://stackoverflow.com/a/63111390
export const getDirFilePaths = async (dirPath: string) => {
if (!(await fs.stat(dirPath)).isDirectory()) {
@ -181,58 +186,3 @@ export const getZipFileStream = async (
});
return readableStream;
};
export const convertBrowserStreamToNode = (
fileStream: ReadableStream<Uint8Array>,
) => {
const reader = fileStream.getReader();
const rs = new Readable();
rs._read = async () => {
try {
const result = await reader.read();
if (!result.done) {
rs.push(Buffer.from(result.value));
} else {
rs.push(null);
return;
}
} catch (e) {
rs.emit("error", e);
}
};
return rs;
};
export async function writeNodeStream(
filePath: string,
fileStream: NodeJS.ReadableStream,
) {
const writeable = createWriteStream(filePath);
fileStream.on("error", (error) => {
writeable.destroy(error); // Close the writable stream with an error
});
fileStream.pipe(writeable);
await new Promise((resolve, reject) => {
writeable.on("finish", resolve);
writeable.on("error", async (e: unknown) => {
if (existsSync(filePath)) {
await fs.unlink(filePath);
}
reject(e);
});
});
}
export async function writeStream(
filePath: string,
fileStream: ReadableStream<Uint8Array>,
) {
const readable = convertBrowserStreamToNode(fileStream);
await writeNodeStream(filePath, readable);
}

59
desktop/src/services/imageProcessor.ts
View file

@ -1,14 +1,18 @@
import { exec } from "child_process";
import util from "util";
import log from "electron-log";
import { existsSync } from "fs";
import * as fs from "node:fs/promises";
import path from "path";
import util from "util";
import { CustomErrors } from "../constants/errors";
import { writeStream } from "../main/fs";
import { isDev } from "../main/general";
import { logError, logErrorSentry } from "../main/log";
import { ElectronFile } from "../types";
import { isPlatform } from "../utils/common/platform";
import { generateTempFilePath } from "../utils/temp";
import { logErrorSentry } from "../main/log";
import { deleteTempFile } from "./ffmpeg";
const shellescape = require("any-shell-escape");
const asyncExec = util.promisify(exec);
@ -80,6 +84,17 @@ function getImageMagickStaticPath() {
export async function convertToJPEG(
fileData: Uint8Array,
filename: string,
): Promise<Uint8Array> {
if (isPlatform("windows")) {
throw Error(CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED);
}
const convertedFileData = await convertToJPEG_(fileData, filename);
return convertedFileData;
}
async function convertToJPEG_(
fileData: Uint8Array,
filename: string,
): Promise<Uint8Array> {
let tempInputFilePath: string;
let tempOutputFilePath: string;
@ -159,6 +174,44 @@ function constructConvertCommand(
}
export async function generateImageThumbnail(
inputFile: File | ElectronFile,
maxDimension: number,
maxSize: number,
): Promise<Uint8Array> {
let inputFilePath = null;
let createdTempInputFile = null;
try {
if (isPlatform("windows")) {
throw Error(
CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED,
);
}
if (!existsSync(inputFile.path)) {
const tempFilePath = await generateTempFilePath(inputFile.name);
await writeStream(tempFilePath, await inputFile.stream());
inputFilePath = tempFilePath;
createdTempInputFile = true;
} else {
inputFilePath = inputFile.path;
}
const thumbnail = await generateImageThumbnail_(
inputFilePath,
maxDimension,
maxSize,
);
return thumbnail;
} finally {
if (createdTempInputFile) {
try {
await deleteTempFile(inputFilePath);
} catch (e) {
logError(e, "failed to deleteTempFile");
}
}
}
}
async function generateImageThumbnail_(
inputFilePath: string,
width: number,
maxSize: number,

88
desktop/src/services/watch.ts
View file

@ -1,11 +1,95 @@
import type { FSWatcher } from "chokidar";
import ElectronLog from "electron-log";
import { watchStore } from "../stores/watch.store";
import { WatchStoreType } from "../types";
import { WatchMapping, WatchStoreType } from "../types";
import { isMappingPresent } from "../utils/watch";
export const addWatchMapping = async (
watcher: FSWatcher,
rootFolderName: string,
folderPath: string,
uploadStrategy: number,
) => {
ElectronLog.log(`Adding watch mapping: ${folderPath}`);
const watchMappings = getWatchMappings();
if (isMappingPresent(watchMappings, folderPath)) {
throw new Error(`Watch mapping already exists`);
}
watcher.add(folderPath);
watchMappings.push({
rootFolderName,
uploadStrategy,
folderPath,
syncedFiles: [],
ignoredFiles: [],
});
setWatchMappings(watchMappings);
};
export const removeWatchMapping = async (
watcher: FSWatcher,
folderPath: string,
) => {
let watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw new Error(`Watch mapping does not exist`);
}
watcher.unwatch(watchMapping.folderPath);
watchMappings = watchMappings.filter(
(mapping) => mapping.folderPath !== watchMapping.folderPath,
);
setWatchMappings(watchMappings);
};
export function updateWatchMappingSyncedFiles(
folderPath: string,
files: WatchMapping["syncedFiles"],
): void {
const watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw Error(`Watch mapping not found`);
}
watchMapping.syncedFiles = files;
setWatchMappings(watchMappings);
}
export function updateWatchMappingIgnoredFiles(
folderPath: string,
files: WatchMapping["ignoredFiles"],
): void {
const watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw Error(`Watch mapping not found`);
}
watchMapping.ignoredFiles = files;
setWatchMappings(watchMappings);
}
export function getWatchMappings() {
const mappings = watchStore.get("mappings") ?? [];
return mappings;
}
export function setWatchMappings(watchMappings: WatchStoreType["mappings"]) {
function setWatchMappings(watchMappings: WatchStoreType["mappings"]) {
watchStore.set("mappings", watchMappings);
}

125
desktop/src/utils/ipcComms.ts
View file

@ -1,125 +0,0 @@
import chokidar from "chokidar";
import {
app,
BrowserWindow,
dialog,
ipcMain,
safeStorage,
shell,
Tray,
} from "electron";
import path from "path";
import { attachIPCHandlers } from "../main/ipc";
import {
muteUpdateNotification,
skipAppUpdate,
updateAndRestart,
} from "../services/appUpdater";
import {
computeImageEmbedding,
computeTextEmbedding,
} from "../services/clipService";
import { deleteTempFile, runFFmpegCmd } from "../services/ffmpeg";
import { getDirFilePaths } from "../services/fs";
import {
convertToJPEG,
generateImageThumbnail,
} from "../services/imageProcessor";
import { generateTempFilePath } from "./temp";
export default function setupIpcComs(
tray: Tray,
mainWindow: BrowserWindow,
watcher: chokidar.FSWatcher,
): void {
attachIPCHandlers();
ipcMain.handle("select-dir", async () => {
const result = await dialog.showOpenDialog({
properties: ["openDirectory"],
});
if (result.filePaths && result.filePaths.length > 0) {
return result.filePaths[0]?.split(path.sep)?.join(path.posix.sep);
}
});
ipcMain.handle("show-upload-files-dialog", async () => {
const files = await dialog.showOpenDialog({
properties: ["openFile", "multiSelections"],
});
return files.filePaths;
});
ipcMain.handle("show-upload-zip-dialog", async () => {
const files = await dialog.showOpenDialog({
properties: ["openFile", "multiSelections"],
filters: [{ name: "Zip File", extensions: ["zip"] }],
});
return files.filePaths;
});
ipcMain.handle("show-upload-dirs-dialog", async () => {
const dir = await dialog.showOpenDialog({
properties: ["openDirectory", "multiSelections"],
});
let files: string[] = [];
for (const dirPath of dir.filePaths) {
files = [...files, ...(await getDirFilePaths(dirPath))];
}
return files;
});
ipcMain.handle("add-watcher", async (_, args: { dir: string }) => {
watcher.add(args.dir);
});
ipcMain.handle("remove-watcher", async (_, args: { dir: string }) => {
watcher.unwatch(args.dir);
});
ipcMain.handle("safeStorage-encrypt", (_, message) => {
return safeStorage.encryptString(message);
});
ipcMain.handle("safeStorage-decrypt", (_, message) => {
return safeStorage.decryptString(message);
});
ipcMain.handle("convert-to-jpeg", (_, fileData, filename) => {
return convertToJPEG(fileData, filename);
});
ipcMain.handle(
"run-ffmpeg-cmd",
(_, cmd, inputFilePath, outputFileName, dontTimeout) => {
return runFFmpegCmd(
cmd,
inputFilePath,
outputFileName,
dontTimeout,
);
},
);
ipcMain.handle("get-temp-file-path", (_, formatSuffix) => {
return generateTempFilePath(formatSuffix);
});
ipcMain.handle("remove-temp-file", (_, tempFilePath: string) => {
return deleteTempFile(tempFilePath);
});
ipcMain.handle(
"generate-image-thumbnail",
(_, fileData, maxDimension, maxSize) => {
return generateImageThumbnail(fileData, maxDimension, maxSize);
},
);
ipcMain.handle("compute-image-embedding", (_, model, inputFilePath) => {
return computeImageEmbedding(model, inputFilePath);
});
ipcMain.handle("compute-text-embedding", (_, model, text) => {
return computeTextEmbedding(model, text);
});
}

8
desktop/src/utils/main.ts
View file

@ -5,10 +5,10 @@ import os from "os";
import path from "path";
import util from "util";
import { rendererURL } from "../main";
import { isDev } from "../main/general";
import { setupAutoUpdater } from "../services/appUpdater";
import autoLauncher from "../services/autoLauncher";
import { getHideDockIconPreference } from "../services/userPreference";
import { isDev } from "../main/general";
import { isPlatform } from "./common/platform";
import { buildContextMenu, buildMenuBar } from "./menu";
const execAsync = util.promisify(require("child_process").exec);
@ -19,7 +19,8 @@ export async function handleUpdates(mainWindow: BrowserWindow) {
setupAutoUpdater(mainWindow);
}
}
export function setupTrayItem(mainWindow: BrowserWindow) {
export const setupTrayItem = (mainWindow: BrowserWindow) => {
const iconName = isPlatform("mac")
? "taskbar-icon-Template.png"
: "taskbar-icon.png";
@ -31,8 +32,7 @@ export function setupTrayItem(mainWindow: BrowserWindow) {
const tray = new Tray(trayIcon);
tray.setToolTip("ente");
tray.setContextMenu(buildContextMenu(mainWindow));
return tray;
}
};
export function handleDownloads(mainWindow: BrowserWindow) {
mainWindow.webContents.session.on("will-download", (_, item) => {

64
desktop/yarn.lock
View file

@ -1530,6 +1530,11 @@ function-bind@^1.1.1:
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==
function-bind@^1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c"
integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==
get-caller-file@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
@ -1575,7 +1580,7 @@ glob-parent@^6.0.2:
dependencies:
is-glob "^4.0.3"
glob@^7.1.3, glob@^7.1.6:
glob@^7.0.0, glob@^7.1.3, glob@^7.1.6:
version "7.2.3"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
@ -1692,6 +1697,13 @@ has@^1.0.3:
dependencies:
function-bind "^1.1.1"
hasown@^2.0.0:
version "2.0.2"
resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003"
integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==
dependencies:
function-bind "^1.1.2"
hosted-git-info@^2.1.4:
version "2.8.9"
resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
@ -1802,6 +1814,11 @@ inherits@2, inherits@^2.0.3:
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
interpret@^1.0.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e"
integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==
is-arrayish@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
@ -1821,6 +1838,13 @@ is-ci@^3.0.0:
dependencies:
ci-info "^3.2.0"
is-core-module@^2.13.0:
version "2.13.1"
resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384"
integrity sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==
dependencies:
hasown "^2.0.0"
is-core-module@^2.9.0:
version "2.9.0"
resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.9.0.tgz#e1c34429cd51c6dd9e09e0799e396e27b19a9c69"
@ -2135,6 +2159,11 @@ minimatch@^5.1.1:
dependencies:
brace-expansion "^2.0.1"
minimist@^1.2.3:
version "1.2.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
minimist@^1.2.6:
version "1.2.6"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
@ -2502,6 +2531,13 @@ readdirp@~3.6.0:
dependencies:
picomatch "^2.2.1"
rechoir@^0.6.2:
version "0.6.2"
resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"
integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==
dependencies:
resolve "^1.1.6"
regenerator-runtime@^0.14.0:
version "0.14.1"
resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f"
@ -2527,6 +2563,15 @@ resolve-from@^4.0.0:
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
resolve@^1.1.6:
version "1.22.8"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
dependencies:
is-core-module "^2.13.0"
path-parse "^1.0.7"
supports-preserve-symlinks-flag "^1.0.0"
resolve@^1.10.0:
version "1.22.1"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177"
@ -2668,6 +2713,23 @@ shell-quote@^1.8.1:
resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.8.1.tgz#6dbf4db75515ad5bac63b4f1894c3a154c766680"
integrity sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==
shelljs@^0.8.5:
version "0.8.5"
resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c"
integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==
dependencies:
glob "^7.0.0"
interpret "^1.0.0"
rechoir "^0.6.2"
shx@^0.3.4:
version "0.3.4"
resolved "https://registry.yarnpkg.com/shx/-/shx-0.3.4.tgz#74289230b4b663979167f94e1935901406e40f02"
integrity sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==
dependencies:
minimist "^1.2.3"
shelljs "^0.8.5"
simple-update-notifier@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz#d70b92bdab7d6d90dfd73931195a30b6e3d7cebb"

4
web/apps/photos/src/components/Upload/Uploader.tsx
View file

@ -530,13 +530,13 @@ export default function Uploader(props: Props) {
) {
await ImportService.setToUploadCollection(collections);
if (zipPaths.current) {
ElectronAPIs.setToUploadFiles(
await ElectronAPIs.setToUploadFiles(
PICKED_UPLOAD_TYPE.ZIPS,
zipPaths.current,
);
zipPaths.current = null;
}
ElectronAPIs.setToUploadFiles(
await ElectronAPIs.setToUploadFiles(
PICKED_UPLOAD_TYPE.FILES,
filesWithCollectionToUploadIn.map(
({ file }) => (file as ElectronFile).path,

6
web/apps/photos/src/components/WatchFolder/index.tsx
View file

@ -28,7 +28,7 @@ export default function WatchFolder({ open, onClose }: Iprops) {
if (!isElectron()) {
return;
}
setMappings(watchFolderService.getWatchMappings());
watchFolderService.getWatchMappings().then((m) => setMappings(m));
}, []);
useEffect(() => {
@ -87,12 +87,12 @@ export default function WatchFolder({ open, onClose }: Iprops) {
uploadStrategy,
);
setInputFolderPath("");
setMappings(watchFolderService.getWatchMappings());
setMappings(await watchFolderService.getWatchMappings());
};
const handleRemoveWatchMapping = async (mapping: WatchMapping) => {
await watchFolderService.removeWatchMapping(mapping.folderPath);
setMappings(watchFolderService.getWatchMappings());
setMappings(await watchFolderService.getWatchMappings());
};
const closeChoiceModal = () => setChoiceModalOpen(false);

17
web/apps/photos/src/services/importService.ts
View file

@ -40,10 +40,10 @@ class ImportService {
if (collections.length === 1) {
collectionName = collections[0].name;
}
ElectronAPIs.setToUploadCollection(collectionName);
await ElectronAPIs.setToUploadCollection(collectionName);
}
updatePendingUploads(files: FileWithCollection[]) {
async updatePendingUploads(files: FileWithCollection[]) {
const filePaths = [];
for (const fileWithCollection of files) {
if (fileWithCollection.isLivePhoto) {
@ -57,13 +57,16 @@ class ImportService {
filePaths.push((fileWithCollection.file as ElectronFile).path);
}
}
ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, filePaths);
await ElectronAPIs.setToUploadFiles(
PICKED_UPLOAD_TYPE.FILES,
filePaths,
);
}
cancelRemainingUploads() {
ElectronAPIs.setToUploadCollection(null);
ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
async cancelRemainingUploads() {
await ElectronAPIs.setToUploadCollection(null);
await ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
await ElectronAPIs.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
}
}

8
web/apps/photos/src/services/upload/uploadManager.ts
View file

@ -180,7 +180,7 @@ class UploadManager {
if (e.message === CustomError.UPLOAD_CANCELLED) {
if (isElectron()) {
this.remainingFiles = [];
ImportService.cancelRemainingUploads();
await ImportService.cancelRemainingUploads();
}
} else {
logError(e, "uploading failed with error");
@ -326,7 +326,7 @@ class UploadManager {
addLogLine(
`post upload action -> fileUploadResult: ${fileUploadResult} uploadedFile present ${!!uploadedFile}`,
);
this.updateElectronRemainingFiles(fileWithCollection);
await this.updateElectronRemainingFiles(fileWithCollection);
switch (fileUploadResult) {
case UPLOAD_RESULT.FAILED:
case UPLOAD_RESULT.BLOCKED:
@ -427,14 +427,14 @@ class UploadManager {
this.setFiles((files) => sortFiles([...files, decryptedFile]));
}
private updateElectronRemainingFiles(
private async updateElectronRemainingFiles(
fileWithCollection: FileWithCollection,
) {
if (isElectron()) {
this.remainingFiles = this.remainingFiles.filter(
(file) => !areFileWithCollectionsSame(file, fileWithCollection),
);
ImportService.updatePendingUploads(this.remainingFiles);
await ImportService.updatePendingUploads(this.remainingFiles);
}
}

2
web/apps/photos/src/services/watchFolder/watchFolderEventHandlers.ts
View file

@ -58,7 +58,7 @@ export async function diskFileRemovedCallback(filePath: string) {
export async function diskFolderRemovedCallback(folderPath: string) {
try {
const mappings = watchFolderService.getWatchMappings();
const mappings = await watchFolderService.getWatchMappings();
const mapping = mappings.find(
(mapping) => mapping.folderPath === folderPath,
);

17
web/apps/photos/src/services/watchFolder/watchFolderService.ts
View file

@ -72,7 +72,7 @@ class watchFolderService {
async getAndSyncDiffOfFiles() {
try {
let mappings = this.getWatchMappings();
let mappings = await this.getWatchMappings();
if (!mappings?.length) {
return;
@ -205,14 +205,13 @@ class watchFolderService {
}
}
getWatchMappings(): WatchMapping[] {
async getWatchMappings(): Promise<WatchMapping[]> {
try {
return ElectronAPIs.getWatchMappings() ?? [];
return (await ElectronAPIs.getWatchMappings()) ?? [];
} catch (e) {
logError(e, "error while getting watch mappings");
return [];
}
return [];
}
private setIsEventRunning(isEventRunning: boolean) {
@ -234,7 +233,7 @@ class watchFolderService {
addLogLine(
`running event type:${event.type} collectionName:${event.collectionName} folderPath:${event.folderPath} , fileCount:${event.files?.length} pathsCount: ${event.paths?.length}`,
);
const mappings = this.getWatchMappings();
const mappings = await this.getWatchMappings();
const mapping = mappings.find(
(mapping) => mapping.folderPath === event.folderPath,
);
@ -380,7 +379,7 @@ class watchFolderService {
...this.currentlySyncedMapping.syncedFiles,
...syncedFiles,
];
ElectronAPIs.updateWatchMappingSyncedFiles(
await ElectronAPIs.updateWatchMappingSyncedFiles(
this.currentlySyncedMapping.folderPath,
this.currentlySyncedMapping.syncedFiles,
);
@ -390,7 +389,7 @@ class watchFolderService {
...this.currentlySyncedMapping.ignoredFiles,
...ignoredFiles,
];
ElectronAPIs.updateWatchMappingIgnoredFiles(
await ElectronAPIs.updateWatchMappingIgnoredFiles(
this.currentlySyncedMapping.folderPath,
this.currentlySyncedMapping.ignoredFiles,
);
@ -505,7 +504,7 @@ class watchFolderService {
this.currentlySyncedMapping.syncedFiles.filter(
(file) => !filePathsToRemove.has(file.path),
);
ElectronAPIs.updateWatchMappingSyncedFiles(
await ElectronAPIs.updateWatchMappingSyncedFiles(
this.currentlySyncedMapping.folderPath,
this.currentlySyncedMapping.syncedFiles,
);
@ -561,7 +560,7 @@ class watchFolderService {
async getCollectionNameAndFolderPath(filePath: string) {
try {
const mappings = this.getWatchMappings();
const mappings = await this.getWatchMappings();
const mapping = mappings.find(
(mapping) =>

12
web/packages/next/next.config.base.js
View file

@ -11,6 +11,7 @@
*/
const cp = require("child_process");
const os = require("os");
/**
* Return the current commit ID if we're running inside a git repository.
@ -18,8 +19,17 @@ const cp = require("child_process");
const gitSHA = () => {
// Allow the command to fail. gitSHA will be an empty string in such cases.
// This allows us to run the build even when we're outside of a git context.
//
// The /dev/null redirection is needed so that we don't print error messages
// if someone is trying to run outside of a git context. Since the way to
// redirect output and ignore failure is different on Windows, the command
// needs to be OS specific.
const command =
os.platform() == "win32"
? "git rev-parse --short HEAD 2> NUL || cd ."
: "git rev-parse --short HEAD 2>/dev/null || true";
const result = cp
.execSync("git rev-parse --short HEAD 2>/dev/null || true", {
.execSync(command, {
cwd: __dirname,
encoding: "utf8",
})

173
web/packages/shared/electron/types.ts
View file

@ -11,6 +11,17 @@ export enum Model {
ONNX_CLIP = "onnx-clip",
}
export enum FILE_PATH_TYPE {
FILES = "files",
ZIPS = "zips",
}
export enum PICKED_UPLOAD_TYPE {
FILES = "files",
FOLDERS = "folders",
ZIPS = "zips",
}
/**
* Extra APIs provided by the Node.js layer when our code is running in Electron
*
@ -78,75 +89,48 @@ export interface ElectronAPIsType {
exists: (path: string) => Promise<boolean>;
};
/** TODO: AUDIT below this */
/*
* TODO: AUDIT below this - Some of the types we use below are not copyable
* across process boundaries, and such functions will (expectedly) fail at
* runtime. For such functions, find an efficient alternative or refactor
* the dataflow.
*/
// - General
registerForegroundEventListener: (onForeground: () => void) => void;
clearElectronStore: () => void;
// - FS legacy
checkExistsAndCreateDir: (dirPath: string) => Promise<void>;
setEncryptionKey: (encryptionKey: string) => Promise<void>;
getEncryptionKey: () => Promise<string>;
// - App update
updateAndRestart: () => void;
skipAppUpdate: (version: string) => void;
muteUpdateNotification: (version: string) => void;
registerUpdateEventListener: (
showUpdateDialog: (updateInfo: AppUpdateInfo) => void,
) => void;
/** TODO: FIXME or migrate below this */
saveStreamToDisk: (
path: string,
fileStream: ReadableStream<any>,
) => Promise<void>;
saveFileToDisk: (path: string, file: any) => Promise<void>;
selectDirectory: () => Promise<string>;
readTextFile: (path: string) => Promise<string>;
showUploadFilesDialog: () => Promise<ElectronFile[]>;
showUploadDirsDialog: () => Promise<ElectronFile[]>;
getPendingUploads: () => Promise<{
files: ElectronFile[];
collectionName: string;
type: string;
}>;
setToUploadFiles: (type: string, filePaths: string[]) => void;
showUploadZipDialog: () => Promise<{
zipPaths: string[];
files: ElectronFile[];
}>;
getElectronFilesFromGoogleZip: (
filePath: string,
) => Promise<ElectronFile[]>;
setToUploadCollection: (collectionName: string) => void;
getDirFiles: (dirPath: string) => Promise<ElectronFile[]>;
getWatchMappings: () => WatchMapping[];
updateWatchMappingSyncedFiles: (
folderPath: string,
files: WatchMapping["syncedFiles"],
) => void;
updateWatchMappingIgnoredFiles: (
folderPath: string,
files: WatchMapping["ignoredFiles"],
) => void;
addWatchMapping: (
collectionName: string,
folderPath: string,
uploadStrategy: number,
) => Promise<void>;
removeWatchMapping: (folderPath: string) => Promise<void>;
registerWatcherFunctions: (
addFile: (file: ElectronFile) => Promise<void>,
removeFile: (path: string) => Promise<void>,
removeFolder: (folderPath: string) => Promise<void>,
) => void;
isFolder: (dirPath: string) => Promise<boolean>;
setEncryptionKey: (encryptionKey: string) => Promise<void>;
getEncryptionKey: () => Promise<string>;
// - Conversion
convertToJPEG: (
fileData: Uint8Array,
filename: string,
) => Promise<Uint8Array>;
generateImageThumbnail: (
inputFile: File | ElectronFile,
maxDimension: number,
maxSize: number,
) => Promise<Uint8Array>;
runFFmpegCmd: (
cmd: string[],
inputFile: File | ElectronFile,
@ -154,18 +138,87 @@ export interface ElectronAPIsType {
dontTimeout?: boolean,
) => Promise<File>;
generateImageThumbnail: (
inputFile: File | ElectronFile,
maxDimension: number,
maxSize: number,
) => Promise<Uint8Array>;
moveFile: (oldPath: string, newPath: string) => Promise<void>;
deleteFolder: (path: string) => Promise<void>;
deleteFile: (path: string) => Promise<void>;
rename: (oldPath: string, newPath: string) => Promise<void>;
// - ML
computeImageEmbedding: (
model: Model,
imageData: Uint8Array,
) => Promise<Float32Array>;
computeTextEmbedding: (model: Model, text: string) => Promise<Float32Array>;
// - File selection
// TODO: Deprecated - use dialogs on the renderer process itself
selectDirectory: () => Promise<string>;
showUploadFilesDialog: () => Promise<ElectronFile[]>;
showUploadDirsDialog: () => Promise<ElectronFile[]>;
showUploadZipDialog: () => Promise<{
zipPaths: string[];
files: ElectronFile[];
}>;
// - Watch
registerWatcherFunctions: (
addFile: (file: ElectronFile) => Promise<void>,
removeFile: (path: string) => Promise<void>,
removeFolder: (folderPath: string) => Promise<void>,
) => void;
addWatchMapping: (
collectionName: string,
folderPath: string,
uploadStrategy: number,
) => Promise<void>;
removeWatchMapping: (folderPath: string) => Promise<void>;
getWatchMappings: () => Promise<WatchMapping[]>;
updateWatchMappingSyncedFiles: (
folderPath: string,
files: WatchMapping["syncedFiles"],
) => Promise<void>;
updateWatchMappingIgnoredFiles: (
folderPath: string,
files: WatchMapping["ignoredFiles"],
) => Promise<void>;
// - FS legacy
checkExistsAndCreateDir: (dirPath: string) => Promise<void>;
saveStreamToDisk: (
path: string,
fileStream: ReadableStream<any>,
) => Promise<void>;
saveFileToDisk: (path: string, file: any) => Promise<void>;
readTextFile: (path: string) => Promise<string>;
isFolder: (dirPath: string) => Promise<boolean>;
moveFile: (oldPath: string, newPath: string) => Promise<void>;
deleteFolder: (path: string) => Promise<void>;
deleteFile: (path: string) => Promise<void>;
rename: (oldPath: string, newPath: string) => Promise<void>;
// - Upload
getPendingUploads: () => Promise<{
files: ElectronFile[];
collectionName: string;
type: string;
}>;
setToUploadFiles: (
/** TODO(MR): This is the actual type */
// type: FILE_PATH_TYPE,
type: PICKED_UPLOAD_TYPE,
filePaths: string[],
) => Promise<void>;
getElectronFilesFromGoogleZip: (
filePath: string,
) => Promise<ElectronFile[]>;
setToUploadCollection: (collectionName: string) => Promise<void>;
getDirFiles: (dirPath: string) => Promise<ElectronFile[]>;
}