0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[desktop] Watch refactoring to get it work with new IPC (#1486)

Browse source
This commit is contained in:
Manav Rathi 2024-04-19 13:09:40 +05:30 committed by GitHub
commit 967ef2e3ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
46 changed files with 1552 additions and 1605 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
desktop/src/main.ts
View file

@ -24,9 +24,10 @@ import { attachFSWatchIPCHandlers, attachIPCHandlers } from "./main/ipc";
import log, { initLogging } from "./main/log";
import { createApplicationMenu, createTrayContextMenu } from "./main/menu";
import { setupAutoUpdater } from "./main/services/app-update";
import autoLauncher from "./main/services/autoLauncher";
import { initWatcher } from "./main/services/chokidar";
import autoLauncher from "./main/services/auto-launcher";
import { createWatcher } from "./main/services/watch";
import { userPreferences } from "./main/stores/user-preferences";
import { migrateLegacyWatchStoreIfNeeded } from "./main/stores/watch";
import { registerStreamProtocol } from "./main/stream";
import { isDev } from "./main/util";
@ -196,8 +197,6 @@ const createMainWindow = async () => {
window.maximize();
}
window.loadURL(rendererURL);
// Open the DevTools automatically when running in dev mode
if (isDev) window.webContents.openDevTools();
@ -296,6 +295,21 @@ const deleteLegacyDiskCacheDirIfExists = async () => {
}
};
/**
* Older versions of our app used to keep a keys.json. It is not needed anymore,
* remove it if it exists.
*
* This code was added March 2024, and can be removed after some time once most
* people have upgraded to newer versions.
*/
const deleteLegacyKeysStoreIfExists = async () => {
const keysStore = path.join(app.getPath("userData"), "keys.json");
if (existsSync(keysStore)) {
log.info(`Removing legacy keys store at ${keysStore}`);
await fs.rm(keysStore);
}
};
const main = () => {
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
@ -311,6 +325,7 @@ const main = () => {
setupRendererServer();
registerPrivilegedSchemes();
increaseDiskCache();
migrateLegacyWatchStoreIfNeeded();
app.on("second-instance", () => {
// Someone tried to run a second instance, we should focus our window.
@ -325,19 +340,26 @@ const main = () => {
//
// Note that some Electron APIs can only be used after this event occurs.
app.on("ready", async () => {
// Create window and prepare for renderer
mainWindow = await createMainWindow();
Menu.setApplicationMenu(await createApplicationMenu(mainWindow));
setupTrayItem(mainWindow);
attachIPCHandlers();
attachFSWatchIPCHandlers(initWatcher(mainWindow));
attachFSWatchIPCHandlers(createWatcher(mainWindow));
registerStreamProtocol();
if (!isDev) setupAutoUpdater(mainWindow);
handleDownloads(mainWindow);
handleExternalLinks(mainWindow);
addAllowOriginHeader(mainWindow);
// Start loading the renderer
mainWindow.loadURL(rendererURL);
// Continue on with the rest of the startup sequence
Menu.setApplicationMenu(await createApplicationMenu(mainWindow));
setupTrayItem(mainWindow);
if (!isDev) setupAutoUpdater(mainWindow);
try {
deleteLegacyDiskCacheDirIfExists();
deleteLegacyKeysStoreIfExists();
} catch (e) {
// Log but otherwise ignore errors during non-critical startup
// actions.

8
desktop/src/main/fs.ts
View file

@ -22,10 +22,8 @@ export const fsReadTextFile = async (filePath: string) =>
export const fsWriteFile = (path: string, contents: string) =>
fs.writeFile(path, contents);
/* TODO: Audit below this */
export const isFolder = async (dirPath: string) => {
export const fsIsDir = async (dirPath: string) => {
if (!existsSync(dirPath)) return false;
const stats = await fs.stat(dirPath);
return stats.isDirectory();
const stat = await fs.stat(dirPath);
return stat.isDirectory();
};

100
desktop/src/main/ipc.ts
View file

@ -10,7 +10,12 @@
import type { FSWatcher } from "chokidar";
import { ipcMain } from "electron/main";
import type { ElectronFile, FILE_PATH_TYPE, FolderWatch } from "../types/ipc";
import type {
CollectionMapping,
ElectronFile,
FolderWatch,
PendingUploads,
} from "../types/ipc";
import {
selectDirectory,
showUploadDirsDialog,
@ -19,13 +24,13 @@ import {
} from "./dialogs";
import {
fsExists,
fsIsDir,
fsMkdirIfNeeded,
fsReadTextFile,
fsRename,
fsRm,
fsRmdir,
fsWriteFile,
isFolder,
} from "./fs";
import { logToDisk } from "./log";
import {
@ -49,16 +54,17 @@ import {
} from "./services/store";
import {
getElectronFilesFromGoogleZip,
getPendingUploads,
setToUploadCollection,
setToUploadFiles,
pendingUploads,
setPendingUploadCollection,
setPendingUploadFiles,
} from "./services/upload";
import {
addWatchMapping,
getWatchMappings,
removeWatchMapping,
updateWatchMappingIgnoredFiles,
updateWatchMappingSyncedFiles,
watchAdd,
watchFindFiles,
watchGet,
watchRemove,
watchUpdateIgnoredFiles,
watchUpdateSyncedFiles,
} from "./services/watch";
import { openDirectory, openLogDirectory } from "./util";
@ -132,6 +138,8 @@ export const attachIPCHandlers = () => {
fsWriteFile(path, contents),
);
ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath));
// - Conversion
ipcMain.handle("convertToJPEG", (_, fileData, filename) =>
@ -183,28 +191,26 @@ export const attachIPCHandlers = () => {
ipcMain.handle("showUploadZipDialog", () => showUploadZipDialog());
// - FS Legacy
ipcMain.handle("isFolder", (_, dirPath: string) => isFolder(dirPath));
// - Upload
ipcMain.handle("getPendingUploads", () => getPendingUploads());
ipcMain.handle("pendingUploads", () => pendingUploads());
ipcMain.handle("setPendingUploadCollection", (_, collectionName: string) =>
setPendingUploadCollection(collectionName),
);
ipcMain.handle(
"setToUploadFiles",
(_, type: FILE_PATH_TYPE, filePaths: string[]) =>
setToUploadFiles(type, filePaths),
"setPendingUploadFiles",
(_, type: PendingUploads["type"], filePaths: string[]) =>
setPendingUploadFiles(type, filePaths),
);
// -
ipcMain.handle("getElectronFilesFromGoogleZip", (_, filePath: string) =>
getElectronFilesFromGoogleZip(filePath),
);
ipcMain.handle("setToUploadCollection", (_, collectionName: string) =>
setToUploadCollection(collectionName),
);
ipcMain.handle("getDirFiles", (_, dirPath: string) => getDirFiles(dirPath));
};
@ -213,42 +219,36 @@ export const attachIPCHandlers = () => {
* watch folder functionality.
*
* It gets passed a {@link FSWatcher} instance which it can then forward to the
* actual handlers.
* actual handlers if they need access to it to do their thing.
*/
export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => {
// - Watch
ipcMain.handle(
"addWatchMapping",
(
_,
collectionName: string,
folderPath: string,
uploadStrategy: number,
) =>
addWatchMapping(
watcher,
collectionName,
folderPath,
uploadStrategy,
),
);
ipcMain.handle("removeWatchMapping", (_, folderPath: string) =>
removeWatchMapping(watcher, folderPath),
);
ipcMain.handle("getWatchMappings", () => getWatchMappings());
ipcMain.handle("watchGet", () => watchGet(watcher));
ipcMain.handle(
"updateWatchMappingSyncedFiles",
(_, folderPath: string, files: FolderWatch["syncedFiles"]) =>
updateWatchMappingSyncedFiles(folderPath, files),
"watchAdd",
(_, folderPath: string, collectionMapping: CollectionMapping) =>
watchAdd(watcher, folderPath, collectionMapping),
);
ipcMain.handle("watchRemove", (_, folderPath: string) =>
watchRemove(watcher, folderPath),
);
ipcMain.handle(
"updateWatchMappingIgnoredFiles",
(_, folderPath: string, files: FolderWatch["ignoredFiles"]) =>
updateWatchMappingIgnoredFiles(folderPath, files),
"watchUpdateSyncedFiles",
(_, syncedFiles: FolderWatch["syncedFiles"], folderPath: string) =>
watchUpdateSyncedFiles(syncedFiles, folderPath),
);
ipcMain.handle(
"watchUpdateIgnoredFiles",
(_, ignoredFiles: FolderWatch["ignoredFiles"], folderPath: string) =>
watchUpdateIgnoredFiles(ignoredFiles, folderPath),
);
ipcMain.handle("watchFindFiles", (_, folderPath: string) =>
watchFindFiles(folderPath),
);
};

2
desktop/src/main/menu.ts
View file

@ -7,7 +7,7 @@ import {
} from "electron";
import { allowWindowClose } from "../main";
import { forceCheckForAppUpdates } from "./services/app-update";
import autoLauncher from "./services/autoLauncher";
import autoLauncher from "./services/auto-launcher";
import { userPreferences } from "./stores/user-preferences";
import { openLogDirectory } from "./util";

19
desktop/src/main/platform.ts
View file

@ -1,19 +0,0 @@
export function isPlatform(platform: "mac" | "windows" | "linux") {
return getPlatform() === platform;
}
export function getPlatform(): "mac" | "windows" | "linux" {
switch (process.platform) {
case "aix":
case "freebsd":
case "linux":
case "openbsd":
case "android":
return "linux";
case "darwin":
case "sunos":
return "mac";
case "win32":
return "windows";
}
}

8
desktop/src/main/services/app-update.ts
View file

@ -1,9 +1,9 @@
import { compareVersions } from "compare-versions";
import { app, BrowserWindow } from "electron";
import { default as electronLog } from "electron-log";
import { autoUpdater } from "electron-updater";
import { app, BrowserWindow } from "electron/main";
import { allowWindowClose } from "../../main";
import { AppUpdateInfo } from "../../types/ipc";
import { AppUpdate } from "../../types/ipc";
import log from "../log";
import { userPreferences } from "../stores/user-preferences";
@ -52,8 +52,8 @@ const checkForUpdatesAndNotify = async (mainWindow: BrowserWindow) => {
return;
}
const showUpdateDialog = (updateInfo: AppUpdateInfo) =>
mainWindow.webContents.send("appUpdateAvailable", updateInfo);
const showUpdateDialog = (update: AppUpdate) =>
mainWindow.webContents.send("appUpdateAvailable", update);
log.debug(() => "Attempting auto update");
autoUpdater.downloadUpdate();

51
desktop/src/main/services/auto-launcher.ts Normal file
View file

@ -0,0 +1,51 @@
import AutoLaunch from "auto-launch";
import { app } from "electron/main";
class AutoLauncher {
/**
* This property will be set and used on Linux and Windows. On macOS,
* there's a separate API
*/
private autoLaunch?: AutoLaunch;
constructor() {
if (process.platform != "darwin") {
this.autoLaunch = new AutoLaunch({
name: "ente",
isHidden: true,
});
}
}
async isEnabled() {
const autoLaunch = this.autoLaunch;
if (autoLaunch) {
return await autoLaunch.isEnabled();
} else {
return app.getLoginItemSettings().openAtLogin;
}
}
async toggleAutoLaunch() {
const isEnabled = await this.isEnabled();
const autoLaunch = this.autoLaunch;
if (autoLaunch) {
if (isEnabled) await autoLaunch.disable();
else await autoLaunch.enable();
} else {
if (isEnabled) app.setLoginItemSettings({ openAtLogin: false });
else app.setLoginItemSettings({ openAtLogin: true });
}
}
async wasAutoLaunched() {
if (this.autoLaunch) {
return app.commandLine.hasSwitch("hidden");
} else {
// TODO(MR): This apparently doesn't work anymore.
return app.getLoginItemSettings().wasOpenedAtLogin;
}
}
}
export default new AutoLauncher();

41
desktop/src/main/services/autoLauncher.ts
View file

@ -1,41 +0,0 @@
import { AutoLauncherClient } from "../../types/main";
import { isPlatform } from "../platform";
import linuxAndWinAutoLauncher from "./autoLauncherClients/linuxAndWinAutoLauncher";
import macAutoLauncher from "./autoLauncherClients/macAutoLauncher";
class AutoLauncher {
private client: AutoLauncherClient;
async init() {
if (isPlatform("linux") || isPlatform("windows")) {
this.client = linuxAndWinAutoLauncher;
} else {
this.client = macAutoLauncher;
}
// migrate old auto launch settings for windows from mac auto launcher to linux and windows auto launcher
if (isPlatform("windows") && (await macAutoLauncher.isEnabled())) {
await macAutoLauncher.toggleAutoLaunch();
await linuxAndWinAutoLauncher.toggleAutoLaunch();
}
}
async isEnabled() {
if (!this.client) {
await this.init();
}
return await this.client.isEnabled();
}
async toggleAutoLaunch() {
if (!this.client) {
await this.init();
}
await this.client.toggleAutoLaunch();
}
async wasAutoLaunched() {
if (!this.client) {
await this.init();
}
return this.client.wasAutoLaunched();
}
}
export default new AutoLauncher();

39
desktop/src/main/services/autoLauncherClients/linuxAndWinAutoLauncher.ts
View file

@ -1,39 +0,0 @@
import AutoLaunch from "auto-launch";
import { app } from "electron";
import { AutoLauncherClient } from "../../../types/main";
const LAUNCHED_AS_HIDDEN_FLAG = "hidden";
class LinuxAndWinAutoLauncher implements AutoLauncherClient {
private instance: AutoLaunch;
constructor() {
const autoLauncher = new AutoLaunch({
name: "ente",
isHidden: true,
});
this.instance = autoLauncher;
}
async isEnabled() {
return await this.instance.isEnabled();
}
async toggleAutoLaunch() {
if (await this.isEnabled()) {
await this.disableAutoLaunch();
} else {
await this.enableAutoLaunch();
}
}
async wasAutoLaunched() {
return app.commandLine.hasSwitch(LAUNCHED_AS_HIDDEN_FLAG);
}
private async disableAutoLaunch() {
await this.instance.disable();
}
private async enableAutoLaunch() {
await this.instance.enable();
}
}
export default new LinuxAndWinAutoLauncher();

28
desktop/src/main/services/autoLauncherClients/macAutoLauncher.ts
View file

@ -1,28 +0,0 @@
import { app } from "electron";
import { AutoLauncherClient } from "../../../types/main";
class MacAutoLauncher implements AutoLauncherClient {
async isEnabled() {
return app.getLoginItemSettings().openAtLogin;
}
async toggleAutoLaunch() {
if (await this.isEnabled()) {
this.disableAutoLaunch();
} else {
this.enableAutoLaunch();
}
}
async wasAutoLaunched() {
return app.getLoginItemSettings().wasOpenedAtLogin;
}
private disableAutoLaunch() {
app.setLoginItemSettings({ openAtLogin: false });
}
private enableAutoLaunch() {
app.setLoginItemSettings({ openAtLogin: true });
}
}
export default new MacAutoLauncher();

45
desktop/src/main/services/chokidar.ts
View file

@ -1,45 +0,0 @@
import chokidar from "chokidar";
import { BrowserWindow } from "electron";
import path from "path";
import log from "../log";
import { getElectronFile } from "./fs";
import { getWatchMappings } from "./watch";
/**
* Convert a file system {@link filePath} that uses the local system specific
* path separators into a path that uses POSIX file separators.
*/
const normalizeToPOSIX = (filePath: string) =>
filePath.split(path.sep).join(path.posix.sep);
export function initWatcher(mainWindow: BrowserWindow) {
const mappings = getWatchMappings();
const folderPaths = mappings.map((mapping) => {
return mapping.folderPath;
});
const watcher = chokidar.watch(folderPaths, {
awaitWriteFinish: true,
});
watcher
.on("add", async (path) => {
mainWindow.webContents.send(
"watch-add",
await getElectronFile(normalizeToPOSIX(path)),
);
})
.on("unlink", (path) => {
mainWindow.webContents.send("watch-unlink", normalizeToPOSIX(path));
})
.on("unlinkDir", (path) => {
mainWindow.webContents.send(
"watch-unlink-dir",
normalizeToPOSIX(path),
);
})
.on("error", (error) => {
log.error("Error while watching files", error);
});
return watcher;
}

13
desktop/src/main/services/fs.ts
View file

@ -91,19 +91,6 @@ export async function getElectronFile(filePath: string): Promise<ElectronFile> {
};
}
export const getValidPaths = (paths: string[]) => {
if (!paths) {
return [] as string[];
}
return paths.filter(async (path) => {
try {
await fs.stat(path).then((stat) => stat.isFile());
} catch (e) {
return false;
}
});
};
export const getZipFileStream = async (
zip: StreamZip.StreamZipAsync,
filePath: string,

15
desktop/src/main/services/imageProcessor.ts
View file

@ -3,7 +3,6 @@ import fs from "node:fs/promises";
import path from "path";
import { CustomErrors, ElectronFile } from "../../types/ipc";
import log from "../log";
import { isPlatform } from "../platform";
import { writeStream } from "../stream";
import { generateTempFilePath } from "../temp";
import { execAsync, isDev } from "../util";
@ -74,9 +73,8 @@ export async function convertToJPEG(
fileData: Uint8Array,
filename: string,
): Promise<Uint8Array> {
if (isPlatform("windows")) {
if (process.platform == "win32")
throw Error(CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED);
}
const convertedFileData = await convertToJPEG_(fileData, filename);
return convertedFileData;
}
@ -123,7 +121,7 @@ function constructConvertCommand(
tempOutputFilePath: string,
) {
let convertCmd: string[];
if (isPlatform("mac")) {
if (process.platform == "darwin") {
convertCmd = SIPS_HEIC_CONVERT_COMMAND_TEMPLATE.map((cmdPart) => {
if (cmdPart === INPUT_PATH_PLACEHOLDER) {
return tempInputFilePath;
@ -133,7 +131,7 @@ function constructConvertCommand(
}
return cmdPart;
});
} else if (isPlatform("linux")) {
} else if (process.platform == "linux") {
convertCmd = IMAGEMAGICK_HEIC_CONVERT_COMMAND_TEMPLATE.map(
(cmdPart) => {
if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) {
@ -162,11 +160,10 @@ export async function generateImageThumbnail(
let inputFilePath = null;
let createdTempInputFile = null;
try {
if (isPlatform("windows")) {
if (process.platform == "win32")
throw Error(
CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED,
);
}
if (!existsSync(inputFile.path)) {
const tempFilePath = await generateTempFilePath(inputFile.name);
await writeStream(tempFilePath, await inputFile.stream());
@ -237,7 +234,7 @@ function constructThumbnailGenerationCommand(
quality: number,
) {
let thumbnailGenerationCmd: string[];
if (isPlatform("mac")) {
if (process.platform == "darwin") {
thumbnailGenerationCmd = SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map(
(cmdPart) => {
if (cmdPart === INPUT_PATH_PLACEHOLDER) {
@ -255,7 +252,7 @@ function constructThumbnailGenerationCommand(
return cmdPart;
},
);
} else if (isPlatform("linux")) {
} else if (process.platform == "linux") {
thumbnailGenerationCmd =
IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map((cmdPart) => {
if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) {

13
desktop/src/main/services/store.ts
View file

@ -1,12 +1,15 @@
import { safeStorage } from "electron/main";
import { keysStore } from "../stores/keys.store";
import { safeStorageStore } from "../stores/safeStorage.store";
import { uploadStatusStore } from "../stores/upload.store";
import { watchStore } from "../stores/watch.store";
import { safeStorageStore } from "../stores/safe-storage";
import { uploadStatusStore } from "../stores/upload-status";
import { watchStore } from "../stores/watch";
/**
* Clear all stores except user preferences.
*
* This is useful to reset state when the user logs out.
*/
export const clearStores = () => {
uploadStatusStore.clear();
keysStore.clear();
safeStorageStore.clear();
watchStore.clear();
};

115
desktop/src/main/services/upload.ts
View file

@ -1,19 +1,23 @@
import StreamZip from "node-stream-zip";
import { existsSync } from "original-fs";
import path from "path";
import { ElectronFile, FILE_PATH_TYPE } from "../../types/ipc";
import { FILE_PATH_KEYS } from "../../types/main";
import { uploadStatusStore } from "../stores/upload.store";
import { getElectronFile, getValidPaths, getZipFileStream } from "./fs";
import { ElectronFile, type PendingUploads } from "../../types/ipc";
import {
uploadStatusStore,
type UploadStatusStore,
} from "../stores/upload-status";
import { getElectronFile, getZipFileStream } from "./fs";
export const getPendingUploads = async () => {
const filePaths = getSavedFilePaths(FILE_PATH_TYPE.FILES);
const zipPaths = getSavedFilePaths(FILE_PATH_TYPE.ZIPS);
export const pendingUploads = async () => {
const collectionName = uploadStatusStore.get("collectionName");
const filePaths = validSavedPaths("files");
const zipPaths = validSavedPaths("zips");
let files: ElectronFile[] = [];
let type: FILE_PATH_TYPE;
let type: PendingUploads["type"];
if (zipPaths.length) {
type = FILE_PATH_TYPE.ZIPS;
type = "zips";
for (const zipPath of zipPaths) {
files = [
...files,
@ -23,9 +27,10 @@ export const getPendingUploads = async () => {
const pendingFilePaths = new Set(filePaths);
files = files.filter((file) => pendingFilePaths.has(file.path));
} else if (filePaths.length) {
type = FILE_PATH_TYPE.FILES;
type = "files";
files = await Promise.all(filePaths.map(getElectronFile));
}
return {
files,
collectionName,
@ -33,16 +38,56 @@ export const getPendingUploads = async () => {
};
};
export const getSavedFilePaths = (type: FILE_PATH_TYPE) => {
const paths =
getValidPaths(
uploadStatusStore.get(FILE_PATH_KEYS[type]) as string[],
) ?? [];
setToUploadFiles(type, paths);
export const validSavedPaths = (type: PendingUploads["type"]) => {
const key = storeKey(type);
const savedPaths = (uploadStatusStore.get(key) as string[]) ?? [];
const paths = savedPaths.filter((p) => existsSync(p));
uploadStatusStore.set(key, paths);
return paths;
};
export const setPendingUploadCollection = (collectionName: string) => {
if (collectionName) uploadStatusStore.set("collectionName", collectionName);
else uploadStatusStore.delete("collectionName");
};
export const setPendingUploadFiles = (
type: PendingUploads["type"],
filePaths: string[],
) => {
const key = storeKey(type);
if (filePaths) uploadStatusStore.set(key, filePaths);
else uploadStatusStore.delete(key);
};
const storeKey = (type: PendingUploads["type"]): keyof UploadStatusStore => {
switch (type) {
case "zips":
return "zipPaths";
case "files":
return "filePaths";
}
};
export const getElectronFilesFromGoogleZip = async (filePath: string) => {
const zip = new StreamZip.async({
file: filePath,
});
const zipName = path.basename(filePath, ".zip");
const entries = await zip.entries();
const files: ElectronFile[] = [];
for (const entry of Object.values(entries)) {
const basename = path.basename(entry.name);
if (entry.isFile && basename.length > 0 && basename[0] !== ".") {
files.push(await getZipEntryAsElectronFile(zipName, zip, entry));
}
}
return files;
};
export async function getZipEntryAsElectronFile(
zipName: string,
zip: StreamZip.StreamZipAsync,
@ -69,39 +114,3 @@ export async function getZipEntryAsElectronFile(
},
};
}
export const setToUploadFiles = (type: FILE_PATH_TYPE, filePaths: string[]) => {
const key = FILE_PATH_KEYS[type];
if (filePaths) {
uploadStatusStore.set(key, filePaths);
} else {
uploadStatusStore.delete(key);
}
};
export const setToUploadCollection = (collectionName: string) => {
if (collectionName) {
uploadStatusStore.set("collectionName", collectionName);
} else {
uploadStatusStore.delete("collectionName");
}
};
export const getElectronFilesFromGoogleZip = async (filePath: string) => {
const zip = new StreamZip.async({
file: filePath,
});
const zipName = path.basename(filePath, ".zip");
const entries = await zip.entries();
const files: ElectronFile[] = [];
for (const entry of Object.values(entries)) {
const basename = path.basename(entry.name);
if (entry.isFile && basename.length > 0 && basename[0] !== ".") {
files.push(await getZipEntryAsElectronFile(zipName, zip, entry));
}
}
return files;
};

212
desktop/src/main/services/watch.ts
View file

@ -1,101 +1,159 @@
import type { FSWatcher } from "chokidar";
import ElectronLog from "electron-log";
import { FolderWatch, WatchStoreType } from "../../types/ipc";
import { watchStore } from "../stores/watch.store";
import chokidar, { type FSWatcher } from "chokidar";
import { BrowserWindow } from "electron/main";
import fs from "node:fs/promises";
import path from "node:path";
import { FolderWatch, type CollectionMapping } from "../../types/ipc";
import { fsIsDir } from "../fs";
import log from "../log";
import { watchStore } from "../stores/watch";
export const addWatchMapping = async (
watcher: FSWatcher,
rootFolderName: string,
folderPath: string,
uploadStrategy: number,
) => {
ElectronLog.log(`Adding watch mapping: ${folderPath}`);
const watchMappings = getWatchMappings();
if (isMappingPresent(watchMappings, folderPath)) {
throw new Error(`Watch mapping already exists`);
/**
* Create and return a new file system watcher.
*
* Internally this uses the watcher from the chokidar package.
*
* @param mainWindow The window handle is used to notify the renderer process of
* pertinent file system events.
*/
export const createWatcher = (mainWindow: BrowserWindow) => {
const send = (eventName: string) => (path: string) =>
mainWindow.webContents.send(eventName, ...eventData(path));
const folderPaths = folderWatches().map((watch) => watch.folderPath);
const watcher = chokidar.watch(folderPaths, {
awaitWriteFinish: true,
});
watcher
.on("add", send("watchAddFile"))
.on("unlink", send("watchRemoveFile"))
.on("unlinkDir", send("watchRemoveDir"))
.on("error", (error) => log.error("Error while watching files", error));
return watcher;
};
const eventData = (path: string): [string, FolderWatch] => {
path = posixPath(path);
const watch = folderWatches().find((watch) =>
path.startsWith(watch.folderPath + "/"),
);
if (!watch) throw new Error(`No folder watch was found for path ${path}`);
return [path, watch];
};
/**
* Convert a file system {@link filePath} that uses the local system specific
* path separators into a path that uses POSIX file separators.
*/
const posixPath = (filePath: string) =>
filePath.split(path.sep).join(path.posix.sep);
export const watchGet = (watcher: FSWatcher) => {
const [valid, deleted] = folderWatches().reduce(
([valid, deleted], watch) => {
(fsIsDir(watch.folderPath) ? valid : deleted).push(watch);
return [valid, deleted];
},
[[], []],
);
if (deleted.length) {
for (const watch of deleted) watchRemove(watcher, watch.folderPath);
setFolderWatches(valid);
}
return valid;
};
watcher.add(folderPath);
const folderWatches = (): FolderWatch[] => watchStore.get("mappings") ?? [];
watchMappings.push({
rootFolderName,
uploadStrategy,
const setFolderWatches = (watches: FolderWatch[]) =>
watchStore.set("mappings", watches);
export const watchAdd = async (
watcher: FSWatcher,
folderPath: string,
collectionMapping: CollectionMapping,
) => {
const watches = folderWatches();
if (!fsIsDir(folderPath))
throw new Error(
`Attempting to add a folder watch for a folder path ${folderPath} that is not an existing directory`,
);
if (watches.find((watch) => watch.folderPath == folderPath))
throw new Error(
`A folder watch with the given folder path ${folderPath} already exists`,
);
watches.push({
folderPath,
collectionMapping,
syncedFiles: [],
ignoredFiles: [],
});
setWatchMappings(watchMappings);
setFolderWatches(watches);
watcher.add(folderPath);
return watches;
};
function isMappingPresent(watchMappings: FolderWatch[], folderPath: string) {
const watchMapping = watchMappings?.find(
(mapping) => mapping.folderPath === folderPath,
export const watchRemove = async (watcher: FSWatcher, folderPath: string) => {
const watches = folderWatches();
const filtered = watches.filter((watch) => watch.folderPath != folderPath);
if (watches.length == filtered.length)
throw new Error(
`Attempting to remove a non-existing folder watch for folder path ${folderPath}`,
);
return !!watchMapping;
}
setFolderWatches(filtered);
watcher.unwatch(folderPath);
return filtered;
};
export const removeWatchMapping = async (
watcher: FSWatcher,
export const watchUpdateSyncedFiles = (
syncedFiles: FolderWatch["syncedFiles"],
folderPath: string,
) => {
let watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw new Error(`Watch mapping does not exist`);
setFolderWatches(
folderWatches().map((watch) => {
if (watch.folderPath == folderPath) {
watch.syncedFiles = syncedFiles;
}
watcher.unwatch(watchMapping.folderPath);
watchMappings = watchMappings.filter(
(mapping) => mapping.folderPath !== watchMapping.folderPath,
return watch;
}),
);
setWatchMappings(watchMappings);
};
export function updateWatchMappingSyncedFiles(
export const watchUpdateIgnoredFiles = (
ignoredFiles: FolderWatch["ignoredFiles"],
folderPath: string,
files: FolderWatch["syncedFiles"],
): void {
const watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
);
if (!watchMapping) {
throw Error(`Watch mapping not found`);
) => {
setFolderWatches(
folderWatches().map((watch) => {
if (watch.folderPath == folderPath) {
watch.ignoredFiles = ignoredFiles;
}
watchMapping.syncedFiles = files;
setWatchMappings(watchMappings);
}
export function updateWatchMappingIgnoredFiles(
folderPath: string,
files: FolderWatch["ignoredFiles"],
): void {
const watchMappings = getWatchMappings();
const watchMapping = watchMappings.find(
(mapping) => mapping.folderPath === folderPath,
return watch;
}),
);
};
if (!watchMapping) {
throw Error(`Watch mapping not found`);
export const watchFindFiles = async (dirPath: string) => {
const items = await fs.readdir(dirPath, { withFileTypes: true });
let paths: string[] = [];
for (const item of items) {
const itemPath = path.posix.join(dirPath, item.name);
if (item.isFile()) {
paths.push(itemPath);
} else if (item.isDirectory()) {
paths = [...paths, ...(await watchFindFiles(itemPath))];
}
watchMapping.ignoredFiles = files;
setWatchMappings(watchMappings);
}
export function getWatchMappings() {
const mappings = watchStore.get("mappings") ?? [];
return mappings;
}
function setWatchMappings(watchMappings: WatchStoreType["mappings"]) {
watchStore.set("mappings", watchMappings);
}
}
return paths;
};

18
desktop/src/main/stores/keys.store.ts
View file

@ -1,18 +0,0 @@
import Store, { Schema } from "electron-store";
import type { KeysStoreType } from "../../types/main";
const keysStoreSchema: Schema<KeysStoreType> = {
AnonymizeUserID: {
type: "object",
properties: {
id: {
type: "string",
},
},
},
};
export const keysStore = new Store({
name: "keys",
schema: keysStoreSchema,
});

7
desktop/src/main/stores/safeStorage.store.ts → desktop/src/main/stores/safe-storage.ts
View file

@ -1,7 +1,10 @@
import Store, { Schema } from "electron-store";
import type { SafeStorageStoreType } from "../../types/main";
const safeStorageSchema: Schema<SafeStorageStoreType> = {
interface SafeStorageStore {
encryptionKey: string;
}
const safeStorageSchema: Schema<SafeStorageStore> = {
encryptionKey: {
type: "string",
},

11
desktop/src/main/stores/upload.store.ts → desktop/src/main/stores/upload-status.ts
View file

@ -1,7 +1,12 @@
import Store, { Schema } from "electron-store";
import type { UploadStoreType } from "../../types/main";
const uploadStoreSchema: Schema<UploadStoreType> = {
export interface UploadStatusStore {
filePaths: string[];
zipPaths: string[];
collectionName: string;
}
const uploadStatusSchema: Schema<UploadStatusStore> = {
filePaths: {
type: "array",
items: {
@ -21,5 +26,5 @@ const uploadStoreSchema: Schema<UploadStoreType> = {
export const uploadStatusStore = new Store({
name: "upload-status",
schema: uploadStoreSchema,
schema: uploadStatusSchema,
});

4
desktop/src/main/stores/user-preferences.ts
View file

@ -1,12 +1,12 @@
import Store, { Schema } from "electron-store";
interface UserPreferencesSchema {
interface UserPreferences {
hideDockIcon: boolean;
skipAppVersion?: string;
muteUpdateNotificationVersion?: string;
}
const userPreferencesSchema: Schema<UserPreferencesSchema> = {
const userPreferencesSchema: Schema<UserPreferences> = {
hideDockIcon: {
type: "boolean",
},

47
desktop/src/main/stores/watch.store.ts
View file

@ -1,47 +0,0 @@
import Store, { Schema } from "electron-store";
import { WatchStoreType } from "../../types/ipc";
const watchStoreSchema: Schema<WatchStoreType> = {
mappings: {
type: "array",
items: {
type: "object",
properties: {
rootFolderName: {
type: "string",
},
uploadStrategy: {
type: "number",
},
folderPath: {
type: "string",
},
syncedFiles: {
type: "array",
items: {
type: "object",
properties: {
path: {
type: "string",
},
id: {
type: "number",
},
},
},
},
ignoredFiles: {
type: "array",
items: {
type: "string",
},
},
},
},
},
};
export const watchStore = new Store({
name: "watch-status",
schema: watchStoreSchema,
});

73
desktop/src/main/stores/watch.ts Normal file
View file

@ -0,0 +1,73 @@
import Store, { Schema } from "electron-store";
import { type FolderWatch } from "../../types/ipc";
import log from "../log";
interface WatchStore {
mappings: FolderWatchWithLegacyFields[];
}
type FolderWatchWithLegacyFields = FolderWatch & {
/** @deprecated Only retained for migration, do not use in other code */
rootFolderName?: string;
/** @deprecated Only retained for migration, do not use in other code */
uploadStrategy?: number;
};
const watchStoreSchema: Schema<WatchStore> = {
mappings: {
type: "array",
items: {
type: "object",
properties: {
rootFolderName: { type: "string" },
collectionMapping: { type: "string" },
uploadStrategy: { type: "number" },
folderPath: { type: "string" },
syncedFiles: {
type: "array",
items: {
type: "object",
properties: {
path: { type: "string" },
uploadedFileID: { type: "number" },
collectionID: { type: "number" },
},
},
},
ignoredFiles: {
type: "array",
items: { type: "string" },
},
},
},
},
};
export const watchStore = new Store({
name: "watch-status",
schema: watchStoreSchema,
});
/**
* Previous versions of the store used to store an integer to indicate the
* collection mapping, migrate these to the new schema if we encounter them.
*/
export const migrateLegacyWatchStoreIfNeeded = () => {
let needsUpdate = false;
const watches = watchStore.get("mappings")?.map((watch) => {
let collectionMapping = watch.collectionMapping;
if (!collectionMapping) {
collectionMapping = watch.uploadStrategy == 1 ? "parent" : "root";
needsUpdate = true;
}
if (watch.rootFolderName) {
delete watch.rootFolderName;
needsUpdate = true;
}
return { ...watch, collectionMapping };
});
if (needsUpdate) {
watchStore.set("mappings", watches);
log.info("Migrated legacy watch store data to new schema");
}
};

14
desktop/src/main/util.ts
View file

@ -56,6 +56,13 @@ export const openDirectory = async (dirPath: string) => {
if (res) throw new Error(`Failed to open directory ${dirPath}: res`);
};
/**
* Open the app's log directory in the system's folder viewer.
*
* @see {@link openDirectory}
*/
export const openLogDirectory = () => openDirectory(logDirectoryPath());
/**
* Return the path where the logs for the app are saved.
*
@ -72,10 +79,3 @@ export const openDirectory = async (dirPath: string) => {
*
*/
const logDirectoryPath = () => app.getPath("logs");
/**
* Open the app's log directory in the system's folder viewer.
*
* @see {@link openDirectory}
*/
export const openLogDirectory = () => openDirectory(logDirectoryPath());

166
desktop/src/preload.ts
View file

@ -40,12 +40,13 @@
import { contextBridge, ipcRenderer } from "electron/renderer";
// While we can't import other code, we can import types since they're just
// needed when compiling and will not be needed / looked around for at runtime.
// needed when compiling and will not be needed or looked around for at runtime.
import type {
AppUpdateInfo,
AppUpdate,
CollectionMapping,
ElectronFile,
FILE_PATH_TYPE,
FolderWatch,
PendingUploads,
} from "./types/ipc";
// - General
@ -77,12 +78,12 @@ const onMainWindowFocus = (cb?: () => void) => {
// - App update
const onAppUpdateAvailable = (
cb?: ((updateInfo: AppUpdateInfo) => void) | undefined,
cb?: ((update: AppUpdate) => void) | undefined,
) => {
ipcRenderer.removeAllListeners("appUpdateAvailable");
if (cb) {
ipcRenderer.on("appUpdateAvailable", (_, updateInfo: AppUpdateInfo) =>
cb(updateInfo),
ipcRenderer.on("appUpdateAvailable", (_, update: AppUpdate) =>
cb(update),
);
}
};
@ -118,6 +119,9 @@ const fsReadTextFile = (path: string): Promise<string> =>
const fsWriteFile = (path: string, contents: string): Promise<void> =>
ipcRenderer.invoke("fsWriteFile", path, contents);
const fsIsDir = (dirPath: string): Promise<boolean> =>
ipcRenderer.invoke("fsIsDir", dirPath);
// - AUDIT below this
// - Conversion
@ -188,82 +192,78 @@ const showUploadZipDialog = (): Promise<{
// - Watch
const registerWatcherFunctions = (
addFile: (file: ElectronFile) => Promise<void>,
removeFile: (path: string) => Promise<void>,
removeFolder: (folderPath: string) => Promise<void>,
) => {
ipcRenderer.removeAllListeners("watch-add");
ipcRenderer.removeAllListeners("watch-unlink");
ipcRenderer.removeAllListeners("watch-unlink-dir");
ipcRenderer.on("watch-add", (_, file: ElectronFile) => addFile(file));
ipcRenderer.on("watch-unlink", (_, filePath: string) =>
removeFile(filePath),
);
ipcRenderer.on("watch-unlink-dir", (_, folderPath: string) =>
removeFolder(folderPath),
const watchGet = (): Promise<FolderWatch[]> => ipcRenderer.invoke("watchGet");
const watchAdd = (
folderPath: string,
collectionMapping: CollectionMapping,
): Promise<FolderWatch[]> =>
ipcRenderer.invoke("watchAdd", folderPath, collectionMapping);
const watchRemove = (folderPath: string): Promise<FolderWatch[]> =>
ipcRenderer.invoke("watchRemove", folderPath);
const watchUpdateSyncedFiles = (
syncedFiles: FolderWatch["syncedFiles"],
folderPath: string,
): Promise<void> =>
ipcRenderer.invoke("watchUpdateSyncedFiles", syncedFiles, folderPath);
const watchUpdateIgnoredFiles = (
ignoredFiles: FolderWatch["ignoredFiles"],
folderPath: string,
): Promise<void> =>
ipcRenderer.invoke("watchUpdateIgnoredFiles", ignoredFiles, folderPath);
const watchOnAddFile = (f: (path: string, watch: FolderWatch) => void) => {
ipcRenderer.removeAllListeners("watchAddFile");
ipcRenderer.on("watchAddFile", (_, path: string, watch: FolderWatch) =>
f(path, watch),
);
};
const addWatchMapping = (
collectionName: string,
folderPath: string,
uploadStrategy: number,
): Promise<void> =>
ipcRenderer.invoke(
"addWatchMapping",
collectionName,
folderPath,
uploadStrategy,
const watchOnRemoveFile = (f: (path: string, watch: FolderWatch) => void) => {
ipcRenderer.removeAllListeners("watchRemoveFile");
ipcRenderer.on("watchRemoveFile", (_, path: string, watch: FolderWatch) =>
f(path, watch),
);
};
const removeWatchMapping = (folderPath: string): Promise<void> =>
ipcRenderer.invoke("removeWatchMapping", folderPath);
const watchOnRemoveDir = (f: (path: string, watch: FolderWatch) => void) => {
ipcRenderer.removeAllListeners("watchRemoveDir");
ipcRenderer.on("watchRemoveDir", (_, path: string, watch: FolderWatch) =>
f(path, watch),
);
};
const getWatchMappings = (): Promise<FolderWatch[]> =>
ipcRenderer.invoke("getWatchMappings");
const updateWatchMappingSyncedFiles = (
folderPath: string,
files: FolderWatch["syncedFiles"],
): Promise<void> =>
ipcRenderer.invoke("updateWatchMappingSyncedFiles", folderPath, files);
const updateWatchMappingIgnoredFiles = (
folderPath: string,
files: FolderWatch["ignoredFiles"],
): Promise<void> =>
ipcRenderer.invoke("updateWatchMappingIgnoredFiles", folderPath, files);
// - FS Legacy
const isFolder = (dirPath: string): Promise<boolean> =>
ipcRenderer.invoke("isFolder", dirPath);
const watchFindFiles = (folderPath: string): Promise<string[]> =>
ipcRenderer.invoke("watchFindFiles", folderPath);
// - Upload
const getPendingUploads = (): Promise<{
files: ElectronFile[];
collectionName: string;
type: string;
}> => ipcRenderer.invoke("getPendingUploads");
const pendingUploads = (): Promise<PendingUploads | undefined> =>
ipcRenderer.invoke("pendingUploads");
const setToUploadFiles = (
type: FILE_PATH_TYPE,
const setPendingUploadCollection = (collectionName: string): Promise<void> =>
ipcRenderer.invoke("setPendingUploadCollection", collectionName);
const setPendingUploadFiles = (
type: PendingUploads["type"],
filePaths: string[],
): Promise<void> => ipcRenderer.invoke("setToUploadFiles", type, filePaths);
): Promise<void> =>
ipcRenderer.invoke("setPendingUploadFiles", type, filePaths);
// -
const getElectronFilesFromGoogleZip = (
filePath: string,
): Promise<ElectronFile[]> =>
ipcRenderer.invoke("getElectronFilesFromGoogleZip", filePath);
const setToUploadCollection = (collectionName: string): Promise<void> =>
ipcRenderer.invoke("setToUploadCollection", collectionName);
const getDirFiles = (dirPath: string): Promise<ElectronFile[]> =>
ipcRenderer.invoke("getDirFiles", dirPath);
//
// These objects exposed here will become available to the JS code in our
// renderer (the web/ code) as `window.ElectronAPIs.*`
//
@ -295,10 +295,13 @@ const getDirFiles = (dirPath: string): Promise<ElectronFile[]> =>
// https://www.electronjs.org/docs/latest/api/context-bridge#methods
//
// The copy itself is relatively fast, but the problem with transfering large
// amounts of data is potentially running out of memory during the copy. For an
// alternative, see [Note: IPC streams].
// amounts of data is potentially running out of memory during the copy.
//
// For an alternative, see [Note: IPC streams].
//
contextBridge.exposeInMainWorld("electron", {
// - General
appVersion,
logToDisk,
openDirectory,
@ -309,12 +312,14 @@ contextBridge.exposeInMainWorld("electron", {
onMainWindowFocus,
// - App update
onAppUpdateAvailable,
updateAndRestart,
updateOnNextRestart,
skipAppUpdate,
// - FS
fs: {
exists: fsExists,
rename: fsRename,
@ -323,42 +328,51 @@ contextBridge.exposeInMainWorld("electron", {
rm: fsRm,
readTextFile: fsReadTextFile,
writeFile: fsWriteFile,
isDir: fsIsDir,
},
// - Conversion
convertToJPEG,
generateImageThumbnail,
runFFmpegCmd,
// - ML
clipImageEmbedding,
clipTextEmbedding,
detectFaces,
faceEmbedding,
// - File selection
selectDirectory,
showUploadFilesDialog,
showUploadDirsDialog,
showUploadZipDialog,
// - Watch
registerWatcherFunctions,
addWatchMapping,
removeWatchMapping,
getWatchMappings,
updateWatchMappingSyncedFiles,
updateWatchMappingIgnoredFiles,
// - FS legacy
// TODO: Move these into fs + document + rename if needed
isFolder,
watch: {
get: watchGet,
add: watchAdd,
remove: watchRemove,
onAddFile: watchOnAddFile,
onRemoveFile: watchOnRemoveFile,
onRemoveDir: watchOnRemoveDir,
findFiles: watchFindFiles,
updateSyncedFiles: watchUpdateSyncedFiles,
updateIgnoredFiles: watchUpdateIgnoredFiles,
},
// - Upload
getPendingUploads,
setToUploadFiles,
pendingUploads,
setPendingUploadCollection,
setPendingUploadFiles,
// -
getElectronFilesFromGoogleZip,
setToUploadCollection,
getDirFiles,
});

31
desktop/src/types/ipc.ts
View file

@ -5,20 +5,32 @@
* See [Note: types.ts <-> preload.ts <-> ipc.ts]
*/
export interface AppUpdate {
autoUpdatable: boolean;
version: string;
}
export interface FolderWatch {
rootFolderName: string;
uploadStrategy: number;
collectionMapping: CollectionMapping;
folderPath: string;
syncedFiles: FolderWatchSyncedFile[];
ignoredFiles: string[];
}
export type CollectionMapping = "root" | "parent";
export interface FolderWatchSyncedFile {
path: string;
uploadedFileID: number;
collectionID: number;
}
export interface PendingUploads {
collectionName: string;
type: "files" | "zips";
files: ElectronFile[];
}
/**
* Errors that have special semantics on the web side.
*
@ -65,18 +77,3 @@ export interface ElectronFile {
blob: () => Promise<Blob>;
arrayBuffer: () => Promise<Uint8Array>;
}
export interface WatchStoreType {
mappings: FolderWatch[];
}
export enum FILE_PATH_TYPE {
/* eslint-disable no-unused-vars */
FILES = "files",
ZIPS = "zips",
}
export interface AppUpdateInfo {
autoUpdatable: boolean;
version: string;
}

31
desktop/src/types/main.ts
View file

@ -1,31 +0,0 @@
import { FILE_PATH_TYPE } from "./ipc";
export interface AutoLauncherClient {
isEnabled: () => Promise<boolean>;
toggleAutoLaunch: () => Promise<void>;
wasAutoLaunched: () => Promise<boolean>;
}
export interface UploadStoreType {
filePaths: string[];
zipPaths: string[];
collectionName: string;
}
export interface KeysStoreType {
AnonymizeUserID: {
id: string;
};
}
/* eslint-disable no-unused-vars */
export const FILE_PATH_KEYS: {
[k in FILE_PATH_TYPE]: keyof UploadStoreType;
} = {
[FILE_PATH_TYPE.ZIPS]: "zipPaths",
[FILE_PATH_TYPE.FILES]: "filePaths",
};
export interface SafeStorageStoreType {
encryptionKey: string;
}

7
web/apps/cast/src/types/upload/index.ts
View file

@ -95,13 +95,6 @@ export interface ParsedExtractedMetadata {
height: number;
}
// This is used to prompt the user the make upload strategy choice
export interface ImportSuggestion {
rootFolderName: string;
hasNestedFolders: boolean;
hasRootLevelFileWithFolder: boolean;
}
export interface PublicUploadProps {
token: string;
passwordToken: string;

7
web/apps/photos/src/components/Sidebar/UtilitySection.tsx
View file

@ -206,7 +206,12 @@ export default function UtilitySection({ closeSidebar }) {
closeSidebar={closeSidebar}
setLoading={startLoading}
/>
<WatchFolder open={watchFolderView} onClose={closeWatchFolder} />
{isElectron() && (
<WatchFolder
open={watchFolderView}
onClose={closeWatchFolder}
/>
)}
<Preferences
open={preferencesView}
onClose={closePreferencesOptions}

32
web/apps/photos/src/components/Upload/UploadStrategyChoiceModal.tsx → web/apps/photos/src/components/Upload/CollectionMappingChoiceModal.tsx
View file

@ -1,3 +1,4 @@
import type { CollectionMapping } from "@/next/types/ipc";
import {
CenteredFlex,
SpaceBetweenFlex,
@ -8,23 +9,19 @@ import DialogTitleWithCloseButton, {
import { Button, Dialog, DialogContent, Typography } from "@mui/material";
import { t } from "i18next";
interface Props {
uploadToMultipleCollection: () => void;
interface CollectionMappingChoiceModalProps {
open: boolean;
onClose: () => void;
uploadToSingleCollection: () => void;
didSelect: (mapping: CollectionMapping) => void;
}
function UploadStrategyChoiceModal({
uploadToMultipleCollection,
uploadToSingleCollection,
...props
}: Props) {
const handleClose = dialogCloseHandler({
onClose: props.onClose,
});
export const CollectionMappingChoiceModal: React.FC<
CollectionMappingChoiceModalProps
> = ({ open, onClose, didSelect }) => {
const handleClose = dialogCloseHandler({ onClose });
return (
<Dialog open={props.open} onClose={handleClose}>
<Dialog open={open} onClose={handleClose}>
<DialogTitleWithCloseButton onClose={handleClose}>
{t("MULTI_FOLDER_UPLOAD")}
</DialogTitleWithCloseButton>
@ -39,8 +36,8 @@ function UploadStrategyChoiceModal({
size="medium"
color="accent"
onClick={() => {
props.onClose();
uploadToSingleCollection();
onClose();
didSelect("root");
}}
>
{t("UPLOAD_STRATEGY_SINGLE_COLLECTION")}
@ -52,8 +49,8 @@ function UploadStrategyChoiceModal({
size="medium"
color="accent"
onClick={() => {
props.onClose();
uploadToMultipleCollection();
onClose();
didSelect("parent");
}}
>
{t("UPLOAD_STRATEGY_COLLECTION_PER_FOLDER")}
@ -62,5 +59,4 @@ function UploadStrategyChoiceModal({
</DialogContent>
</Dialog>
);
}
export default UploadStrategyChoiceModal;
};

120
web/apps/photos/src/components/Upload/Uploader.tsx
View file

@ -1,15 +1,11 @@
import { ensureElectron } from "@/next/electron";
import log from "@/next/log";
import type { Electron } from "@/next/types/ipc";
import type { CollectionMapping, Electron } from "@/next/types/ipc";
import { CustomError } from "@ente/shared/error";
import { isPromise } from "@ente/shared/utils";
import DiscFullIcon from "@mui/icons-material/DiscFull";
import UserNameInputDialog from "components/UserNameInputDialog";
import {
DEFAULT_IMPORT_SUGGESTION,
PICKED_UPLOAD_TYPE,
UPLOAD_STAGES,
UPLOAD_STRATEGY,
} from "constants/upload";
import { PICKED_UPLOAD_TYPE, UPLOAD_STAGES } from "constants/upload";
import { t } from "i18next";
import isElectron from "is-electron";
import { AppContext } from "pages/_app";
@ -17,14 +13,14 @@ import { GalleryContext } from "pages/gallery";
import { useContext, useEffect, useRef, useState } from "react";
import billingService from "services/billingService";
import { getLatestCollections } from "services/collectionService";
import ImportService from "services/importService";
import { setToUploadCollection } from "services/pending-uploads";
import {
getPublicCollectionUID,
getPublicCollectionUploaderName,
savePublicCollectionUploaderName,
} from "services/publicCollectionService";
import uploadManager from "services/upload/uploadManager";
import watchFolderService from "services/watch";
import watcher from "services/watch";
import { NotificationAttributes } from "types/Notification";
import { Collection } from "types/collection";
import {
@ -35,11 +31,7 @@ import {
SetLoading,
UploadTypeSelectorIntent,
} from "types/gallery";
import {
ElectronFile,
FileWithCollection,
ImportSuggestion,
} from "types/upload";
import { ElectronFile, FileWithCollection } from "types/upload";
import {
InProgressUpload,
SegregatedFinishedUploads,
@ -53,13 +45,15 @@ import {
getRootLevelFileWithFolderNotAllowMessage,
} from "utils/ui";
import {
DEFAULT_IMPORT_SUGGESTION,
filterOutSystemFiles,
getImportSuggestion,
groupFilesBasedOnParentFolder,
type ImportSuggestion,
} from "utils/upload";
import { SetCollectionNamerAttributes } from "../Collections/CollectionNamer";
import { CollectionMappingChoiceModal } from "./CollectionMappingChoiceModal";
import UploadProgress from "./UploadProgress";
import UploadStrategyChoiceModal from "./UploadStrategyChoiceModal";
import UploadTypeSelector from "./UploadTypeSelector";
const FIRST_ALBUM_NAME = "My First Album";
@ -137,6 +131,7 @@ export default function Uploader(props: Props) {
const closeUploadProgress = () => setUploadProgressView(false);
const showUserNameInputDialog = () => setUserNameInputDialogView(true);
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const setCollectionName = (collectionName: string) => {
isPendingDesktopUpload.current = true;
pendingDesktopUploadCollectionName.current = collectionName;
@ -177,18 +172,27 @@ export default function Uploader(props: Props) {
}
if (isElectron()) {
ImportService.getPendingUploads().then(
({ files: electronFiles, collectionName, type }) => {
log.info(`found pending desktop upload, resuming uploads`);
resumeDesktopUpload(type, electronFiles, collectionName);
},
ensureElectron()
.pendingUploads()
.then((pending) => {
if (pending) {
log.info("Resuming pending desktop upload", pending);
resumeDesktopUpload(
pending.type == "files"
? PICKED_UPLOAD_TYPE.FILES
: PICKED_UPLOAD_TYPE.ZIPS,
pending.files,
pending.collectionName,
);
watchFolderService.init(
}
});
/* TODO(MR): This is the connection point, implement
watcher.init(
setElectronFiles,
setCollectionName,
props.syncWithRemote,
appContext.setIsFolderSyncRunning,
);
*/
}
}, [
publicCollectionGalleryContext.accessedThroughSharedURL,
@ -291,18 +295,16 @@ export default function Uploader(props: Props) {
}`,
);
if (uploadManager.isUploadRunning()) {
if (watchFolderService.isUploadRunning()) {
if (watcher.isUploadRunning()) {
// Pause watch folder sync on user upload
log.info(
"watchFolder upload was running, pausing it to run user upload",
"Folder watcher was uploading, pausing it to first run user upload",
);
// pause watch folder service on user upload
watchFolderService.pauseRunningSync();
watcher.pauseRunningSync();
} else {
log.info(
"an upload is already running, rejecting new upload request",
"Ignoring new upload request because an upload is already running",
);
// no-op
// a user upload is already in progress
return;
}
}
@ -330,7 +332,7 @@ export default function Uploader(props: Props) {
const importSuggestion = getImportSuggestion(
pickedUploadType.current,
toUploadFiles.current,
toUploadFiles.current.map((file) => file["path"]),
);
setImportSuggestion(importSuggestion);
@ -391,7 +393,7 @@ export default function Uploader(props: Props) {
};
const uploadFilesToNewCollections = async (
strategy: UPLOAD_STRATEGY,
strategy: CollectionMapping,
collectionName?: string,
) => {
try {
@ -405,7 +407,7 @@ export default function Uploader(props: Props) {
string,
(File | ElectronFile)[]
>();
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
if (strategy == "root") {
collectionNameToFilesMap.set(
collectionName,
toUploadFiles.current,
@ -505,18 +507,19 @@ export default function Uploader(props: Props) {
if (
electron &&
!isPendingDesktopUpload.current &&
!watchFolderService.isUploadRunning()
!watcher.isUploadRunning()
) {
await ImportService.setToUploadCollection(collections);
await setToUploadCollection(collections);
// TODO (MR): What happens when we have both?
if (zipPaths.current) {
await electron.setToUploadFiles(
PICKED_UPLOAD_TYPE.ZIPS,
await electron.setPendingUploadFiles(
"zips",
zipPaths.current,
);
zipPaths.current = null;
}
await electron.setToUploadFiles(
PICKED_UPLOAD_TYPE.FILES,
await electron.setPendingUploadFiles(
"files",
filesWithCollectionToUploadIn.map(
({ file }) => (file as ElectronFile).path,
),
@ -532,14 +535,14 @@ export default function Uploader(props: Props) {
closeUploadProgress();
}
if (isElectron()) {
if (watchFolderService.isUploadRunning()) {
await watchFolderService.allFileUploadsDone(
if (watcher.isUploadRunning()) {
await watcher.allFileUploadsDone(
filesWithCollectionToUploadIn,
collections,
);
} else if (watchFolderService.isSyncPaused()) {
} else if (watcher.isSyncPaused()) {
// resume the service after user upload is done
watchFolderService.resumePausedSync();
watcher.resumePausedSync();
}
}
} catch (e) {
@ -605,10 +608,7 @@ export default function Uploader(props: Props) {
}
const uploadToSingleNewCollection = (collectionName: string) => {
uploadFilesToNewCollections(
UPLOAD_STRATEGY.SINGLE_COLLECTION,
collectionName,
);
uploadFilesToNewCollections("root", collectionName);
};
const showCollectionCreateModal = (suggestedName: string) => {
@ -647,7 +647,7 @@ export default function Uploader(props: Props) {
`upload pending files to collection - ${pendingDesktopUploadCollectionName.current}`,
);
uploadFilesToNewCollections(
UPLOAD_STRATEGY.SINGLE_COLLECTION,
"root",
pendingDesktopUploadCollectionName.current,
);
pendingDesktopUploadCollectionName.current = null;
@ -655,17 +655,13 @@ export default function Uploader(props: Props) {
log.info(
`pending upload - strategy - "multiple collections" `,
);
uploadFilesToNewCollections(
UPLOAD_STRATEGY.COLLECTION_PER_FOLDER,
);
uploadFilesToNewCollections("parent");
}
return;
}
if (isElectron() && pickedUploadType === PICKED_UPLOAD_TYPE.ZIPS) {
log.info("uploading zip files");
uploadFilesToNewCollections(
UPLOAD_STRATEGY.COLLECTION_PER_FOLDER,
);
uploadFilesToNewCollections("parent");
return;
}
if (isFirstUpload && !importSuggestion.rootFolderName) {
@ -784,16 +780,26 @@ export default function Uploader(props: Props) {
);
return;
}
uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
uploadFilesToNewCollections("parent");
};
const didSelectCollectionMapping = (mapping: CollectionMapping) => {
switch (mapping) {
case "root":
handleUploadToSingleCollection();
break;
case "parent":
handleUploadToMultipleCollections();
break;
}
};
return (
<>
<UploadStrategyChoiceModal
<CollectionMappingChoiceModal
open={choiceModalView}
onClose={handleChoiceModalClose}
uploadToSingleCollection={handleUploadToSingleCollection}
uploadToMultipleCollection={handleUploadToMultipleCollections}
didSelect={didSelectCollectionMapping}
/>
<UploadTypeSelector
show={props.uploadTypeSelectorView}

242
web/apps/photos/src/components/WatchFolder.tsx
View file

@ -1,3 +1,7 @@
import { ensureElectron } from "@/next/electron";
import { basename } from "@/next/file";
import type { CollectionMapping, FolderWatch } from "@/next/types/ipc";
import { ensure } from "@/utils/ensure";
import {
FlexWrapper,
HorizontalFlex,
@ -23,31 +27,39 @@ import {
Typography,
} from "@mui/material";
import { styled } from "@mui/material/styles";
import UploadStrategyChoiceModal from "components/Upload/UploadStrategyChoiceModal";
import { PICKED_UPLOAD_TYPE, UPLOAD_STRATEGY } from "constants/upload";
import { CollectionMappingChoiceModal } from "components/Upload/CollectionMappingChoiceModal";
import { t } from "i18next";
import { AppContext } from "pages/_app";
import React, { useContext, useEffect, useState } from "react";
import watchFolderService from "services/watch";
import { WatchMapping } from "types/watchFolder";
import { getImportSuggestion } from "utils/upload";
import watcher from "services/watch";
import { areAllInSameDirectory } from "utils/upload";
interface WatchFolderProps {
open: boolean;
onClose: () => void;
}
/**
* View the state of and manage folder watches.
*
* This is the screen that controls that "watch folder" feature in the app.
*/
export const WatchFolder: React.FC<WatchFolderProps> = ({ open, onClose }) => {
const [mappings, setMappings] = useState<WatchMapping[]>([]);
const [inputFolderPath, setInputFolderPath] = useState("");
// The folders we are watching
const [watches, setWatches] = useState<FolderWatch[] | undefined>();
// Temporarily stash the folder path while we show a choice dialog to the
// user to select the collection mapping.
const [savedFolderPath, setSavedFolderPath] = useState<
string | undefined
>();
// True when we're showing the choice dialog to ask the user to set the
// collection mapping.
const [choiceModalOpen, setChoiceModalOpen] = useState(false);
const appContext = useContext(AppContext);
const electron = globalThis.electron;
useEffect(() => {
if (!electron) return;
watchFolderService.getWatchMappings().then((m) => setMappings(m));
watcher.getWatches().then((ws) => setWatches(ws));
}, []);
useEffect(() => {
@ -64,69 +76,41 @@ export const WatchFolder: React.FC<WatchFolderProps> = ({ open, onClose }) => {
for (let i = 0; i < folders.length; i++) {
const folder: any = folders[i];
const path = (folder.path as string).replace(/\\/g, "/");
if (await watchFolderService.isFolder(path)) {
await addFolderForWatching(path);
if (await ensureElectron().fs.isDir(path)) {
await selectCollectionMappingAndAddWatch(path);
}
}
};
const addFolderForWatching = async (path: string) => {
if (!electron) return;
setInputFolderPath(path);
const files = await electron.getDirFiles(path);
const analysisResult = getImportSuggestion(
PICKED_UPLOAD_TYPE.FOLDERS,
files,
);
if (analysisResult.hasNestedFolders) {
setChoiceModalOpen(true);
const selectCollectionMappingAndAddWatch = async (path: string) => {
const filePaths = await ensureElectron().watch.findFiles(path);
if (areAllInSameDirectory(filePaths)) {
addWatch(path, "root");
} else {
handleAddWatchMapping(UPLOAD_STRATEGY.SINGLE_COLLECTION, path);
setSavedFolderPath(path);
setChoiceModalOpen(true);
}
};
const handleAddFolderClick = async () => {
await handleFolderSelection();
};
const addWatch = (folderPath: string, mapping: CollectionMapping) =>
watcher.addWatch(folderPath, mapping).then((ws) => setWatches(ws));
const handleFolderSelection = async () => {
const folderPath = await watchFolderService.selectFolder();
if (folderPath) {
await addFolderForWatching(folderPath);
const addNewWatch = async () => {
const dirPath = await ensureElectron().selectDirectory();
if (dirPath) {
await selectCollectionMappingAndAddWatch(dirPath);
}
};
const handleAddWatchMapping = async (
uploadStrategy: UPLOAD_STRATEGY,
folderPath?: string,
) => {
folderPath = folderPath || inputFolderPath;
await watchFolderService.addWatchMapping(
folderPath.substring(folderPath.lastIndexOf("/") + 1),
folderPath,
uploadStrategy,
);
setInputFolderPath("");
setMappings(await watchFolderService.getWatchMappings());
};
const handleRemoveWatchMapping = (mapping: WatchMapping) => {
watchFolderService
.mappingsAfterRemovingFolder(mapping.folderPath)
.then((ms) => setMappings(ms));
};
const removeWatch = async (watch: FolderWatch) =>
watcher.removeWatch(watch.folderPath).then((ws) => setWatches(ws));
const closeChoiceModal = () => setChoiceModalOpen(false);
const uploadToSingleCollection = () => {
const addWatchWithMapping = (mapping: CollectionMapping) => {
closeChoiceModal();
handleAddWatchMapping(UPLOAD_STRATEGY.SINGLE_COLLECTION);
};
const uploadToMultipleCollection = () => {
closeChoiceModal();
handleAddWatchMapping(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER);
setSavedFolderPath(undefined);
addWatch(ensure(savedFolderPath), mapping);
};
return (
@ -144,15 +128,8 @@ export const WatchFolder: React.FC<WatchFolderProps> = ({ open, onClose }) => {
</DialogTitleWithCloseButton>
<DialogContent sx={{ flex: 1 }}>
<Stack spacing={1} p={1.5} height={"100%"}>
<MappingList
mappings={mappings}
handleRemoveWatchMapping={handleRemoveWatchMapping}
/>
<Button
fullWidth
color="accent"
onClick={handleAddFolderClick}
>
<WatchList {...{ watches, removeWatch }} />
<Button fullWidth color="accent" onClick={addNewWatch}>
<span>+</span>
<span
style={{
@ -164,17 +141,39 @@ export const WatchFolder: React.FC<WatchFolderProps> = ({ open, onClose }) => {
</Stack>
</DialogContent>
</Dialog>
<UploadStrategyChoiceModal
<CollectionMappingChoiceModal
open={choiceModalOpen}
onClose={closeChoiceModal}
uploadToSingleCollection={uploadToSingleCollection}
uploadToMultipleCollection={uploadToMultipleCollection}
didSelect={addWatchWithMapping}
/>
</>
);
};
const MappingsContainer = styled(Box)(() => ({
interface WatchList {
watches: FolderWatch[];
removeWatch: (watch: FolderWatch) => void;
}
const WatchList: React.FC<WatchList> = ({ watches, removeWatch }) => {
return watches.length === 0 ? (
<NoWatches />
) : (
<WatchesContainer>
{watches.map((watch) => {
return (
<WatchEntry
key={watch.folderPath}
watch={watch}
removeWatch={removeWatch}
/>
);
})}
</WatchesContainer>
);
};
const WatchesContainer = styled(Box)(() => ({
height: "278px",
overflow: "auto",
"&::-webkit-scrollbar": {
@ -182,47 +181,9 @@ const MappingsContainer = styled(Box)(() => ({
},
}));
const NoMappingsContainer = styled(VerticallyCentered)({
textAlign: "left",
alignItems: "flex-start",
marginBottom: "32px",
});
const EntryContainer = styled(Box)({
marginLeft: "12px",
marginRight: "6px",
marginBottom: "12px",
});
interface MappingListProps {
mappings: WatchMapping[];
handleRemoveWatchMapping: (value: WatchMapping) => void;
}
const MappingList: React.FC<MappingListProps> = ({
mappings,
handleRemoveWatchMapping,
}) => {
return mappings.length === 0 ? (
<NoMappingsContent />
) : (
<MappingsContainer>
{mappings.map((mapping) => {
const NoWatches: React.FC = () => {
return (
<MappingEntry
key={mapping.rootFolderName}
mapping={mapping}
handleRemoveMapping={handleRemoveWatchMapping}
/>
);
})}
</MappingsContainer>
);
};
const NoMappingsContent: React.FC = () => {
return (
<NoMappingsContainer>
<NoWatchesContainer>
<Stack spacing={1}>
<Typography variant="large" fontWeight={"bold"}>
{t("NO_FOLDERS_ADDED")}
@ -243,10 +204,16 @@ const NoMappingsContent: React.FC = () => {
</FlexWrapper>
</Typography>
</Stack>
</NoMappingsContainer>
</NoWatchesContainer>
);
};
const NoWatchesContainer = styled(VerticallyCentered)({
textAlign: "left",
alignItems: "flex-start",
marginBottom: "32px",
});
const CheckmarkIcon: React.FC = () => {
return (
<CheckIcon
@ -254,28 +221,20 @@ const CheckmarkIcon: React.FC = () => {
sx={{
display: "inline",
fontSize: "15px",
color: (theme) => theme.palette.secondary.main,
}}
/>
);
};
interface MappingEntryProps {
mapping: WatchMapping;
handleRemoveMapping: (mapping: WatchMapping) => void;
interface WatchEntryProps {
watch: FolderWatch;
removeWatch: (watch: FolderWatch) => void;
}
const MappingEntry: React.FC<MappingEntryProps> = ({
mapping,
handleRemoveMapping,
}) => {
const WatchEntry: React.FC<WatchEntryProps> = ({ watch, removeWatch }) => {
const appContext = React.useContext(AppContext);
const stopWatching = () => {
handleRemoveMapping(mapping);
};
const confirmStopWatching = () => {
appContext.setDialogMessage({
title: t("STOP_WATCHING_FOLDER"),
@ -285,7 +244,7 @@ const MappingEntry: React.FC<MappingEntryProps> = ({
variant: "secondary",
},
proceed: {
action: stopWatching,
action: () => removeWatch(watch),
text: t("YES_STOP"),
variant: "critical",
},
@ -295,8 +254,7 @@ const MappingEntry: React.FC<MappingEntryProps> = ({
return (
<SpaceBetweenFlex>
<HorizontalFlex>
{mapping &&
mapping.uploadStrategy === UPLOAD_STRATEGY.SINGLE_COLLECTION ? (
{watch.collectionMapping === "root" ? (
<Tooltip title={t("UPLOADED_TO_SINGLE_COLLECTION")}>
<FolderOpenIcon />
</Tooltip>
@ -306,41 +264,45 @@ const MappingEntry: React.FC<MappingEntryProps> = ({
</Tooltip>
)}
<EntryContainer>
<EntryHeading mapping={mapping} />
<EntryHeading watch={watch} />
<Typography color="text.muted" variant="small">
{mapping.folderPath}
{watch.folderPath}
</Typography>
</EntryContainer>
</HorizontalFlex>
<MappingEntryOptions confirmStopWatching={confirmStopWatching} />
<EntryOptions {...{ confirmStopWatching }} />
</SpaceBetweenFlex>
);
};
const EntryContainer = styled(Box)({
marginLeft: "12px",
marginRight: "6px",
marginBottom: "12px",
});
interface EntryHeadingProps {
mapping: WatchMapping;
watch: FolderWatch;
}
const EntryHeading: React.FC<EntryHeadingProps> = ({ mapping }) => {
const appContext = useContext(AppContext);
const EntryHeading: React.FC<EntryHeadingProps> = ({ watch }) => {
const folderPath = watch.folderPath;
return (
<FlexWrapper gap={1}>
<Typography>{mapping.rootFolderName}</Typography>
{appContext.isFolderSyncRunning &&
watchFolderService.isMappingSyncInProgress(mapping) && (
<Typography>{basename(folderPath)}</Typography>
{watcher.isSyncingFolder(folderPath) && (
<CircularProgress size={12} />
)}
</FlexWrapper>
);
};
interface MappingEntryOptionsProps {
interface EntryOptionsProps {
confirmStopWatching: () => void;
}
const MappingEntryOptions: React.FC<MappingEntryOptionsProps> = ({
confirmStopWatching,
}) => {
const EntryOptions: React.FC<EntryOptionsProps> = ({ confirmStopWatching }) => {
return (
<OverflowMenu
menuPaperProps={{

13
web/apps/photos/src/constants/upload.ts
View file

@ -1,11 +1,6 @@
import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/constants";
import { FILE_TYPE } from "constants/file";
import {
FileTypeInfo,
ImportSuggestion,
Location,
ParsedExtractedMetadata,
} from "types/upload";
import { FileTypeInfo, Location, ParsedExtractedMetadata } from "types/upload";
// list of format that were missed by type-detection for some files.
export const WHITELISTED_FILE_FORMATS: FileTypeInfo[] = [
@ -111,12 +106,6 @@ export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
export const A_SEC_IN_MICROSECONDS = 1e6;
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
rootFolderName: "",
hasNestedFolders: false,
hasRootLevelFileWithFolder: false,
};
export const BLACK_THUMBNAIL_BASE64 =
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEB" +
"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQ" +

15
web/apps/photos/src/pages/_app.tsx
View file

@ -5,7 +5,7 @@ import {
logStartupBanner,
logUnhandledErrorsAndRejections,
} from "@/next/log-web";
import { AppUpdateInfo } from "@/next/types/ipc";
import { AppUpdate } from "@/next/types/ipc";
import {
APPS,
APP_TITLES,
@ -91,8 +91,6 @@ type AppContextType = {
closeMessageDialog: () => void;
setDialogMessage: SetDialogBoxAttributes;
setNotificationAttributes: SetNotificationAttributes;
isFolderSyncRunning: boolean;
setIsFolderSyncRunning: (isRunning: boolean) => void;
watchFolderView: boolean;
setWatchFolderView: (isOpen: boolean) => void;
watchFolderFiles: FileList;
@ -128,7 +126,6 @@ export default function App({ Component, pageProps }: AppProps) {
useState<DialogBoxAttributes>(null);
const [messageDialogView, setMessageDialogView] = useState(false);
const [dialogBoxV2View, setDialogBoxV2View] = useState(false);
const [isFolderSyncRunning, setIsFolderSyncRunning] = useState(false);
const [watchFolderView, setWatchFolderView] = useState(false);
const [watchFolderFiles, setWatchFolderFiles] = useState<FileList>(null);
const isMobile = useMediaQuery("(max-width:428px)");
@ -160,9 +157,9 @@ export default function App({ Component, pageProps }: AppProps) {
const electron = globalThis.electron;
if (!electron) return;
const showUpdateDialog = (updateInfo: AppUpdateInfo) => {
if (updateInfo.autoUpdatable) {
setDialogMessage(getUpdateReadyToInstallMessage(updateInfo));
const showUpdateDialog = (update: AppUpdate) => {
if (update.autoUpdatable) {
setDialogMessage(getUpdateReadyToInstallMessage(update));
} else {
setNotificationAttributes({
endIcon: <ArrowForward />,
@ -170,7 +167,7 @@ export default function App({ Component, pageProps }: AppProps) {
message: t("UPDATE_AVAILABLE"),
onClick: () =>
setDialogMessage(
getUpdateAvailableForDownloadMessage(updateInfo),
getUpdateAvailableForDownloadMessage(update),
),
});
}
@ -403,8 +400,6 @@ export default function App({ Component, pageProps }: AppProps) {
finishLoading,
closeMessageDialog,
setDialogMessage,
isFolderSyncRunning,
setIsFolderSyncRunning,
watchFolderView,
setWatchFolderView,
watchFolderFiles,

74
web/apps/photos/src/services/importService.ts
View file

@ -1,74 +0,0 @@
import { ensureElectron } from "@/next/electron";
import log from "@/next/log";
import { PICKED_UPLOAD_TYPE } from "constants/upload";
import { Collection } from "types/collection";
import { ElectronFile, FileWithCollection } from "types/upload";
interface PendingUploads {
files: ElectronFile[];
collectionName: string;
type: PICKED_UPLOAD_TYPE;
}
class ImportService {
async getPendingUploads(): Promise<PendingUploads> {
try {
const pendingUploads =
(await ensureElectron().getPendingUploads()) as PendingUploads;
return pendingUploads;
} catch (e) {
if (e?.message?.includes("ENOENT: no such file or directory")) {
// ignore
} else {
log.error("failed to getPendingUploads ", e);
}
return { files: [], collectionName: null, type: null };
}
}
async setToUploadCollection(collections: Collection[]) {
let collectionName: string = null;
/* collection being one suggest one of two things
1. Either the user has upload to a single existing collection
2. Created a new single collection to upload to
may have had multiple folder, but chose to upload
to one album
hence saving the collection name when upload collection count is 1
helps the info of user choosing this options
and on next upload we can directly start uploading to this collection
*/
if (collections.length === 1) {
collectionName = collections[0].name;
}
await ensureElectron().setToUploadCollection(collectionName);
}
async updatePendingUploads(files: FileWithCollection[]) {
const filePaths = [];
for (const fileWithCollection of files) {
if (fileWithCollection.isLivePhoto) {
filePaths.push(
(fileWithCollection.livePhotoAssets.image as ElectronFile)
.path,
(fileWithCollection.livePhotoAssets.video as ElectronFile)
.path,
);
} else {
filePaths.push((fileWithCollection.file as ElectronFile).path);
}
}
await ensureElectron().setToUploadFiles(
PICKED_UPLOAD_TYPE.FILES,
filePaths,
);
}
async cancelRemainingUploads() {
const electron = ensureElectron();
await electron.setToUploadCollection(null);
await electron.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []);
await electron.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []);
}
}
export default new ImportService();

42
web/apps/photos/src/services/pending-uploads.ts Normal file
View file

@ -0,0 +1,42 @@
import { ensureElectron } from "@/next/electron";
import { Collection } from "types/collection";
import { ElectronFile, FileWithCollection } from "types/upload";
export const setToUploadCollection = async (collections: Collection[]) => {
let collectionName: string = null;
/* collection being one suggest one of two things
1. Either the user has upload to a single existing collection
2. Created a new single collection to upload to
may have had multiple folder, but chose to upload
to one album
hence saving the collection name when upload collection count is 1
helps the info of user choosing this options
and on next upload we can directly start uploading to this collection
*/
if (collections.length === 1) {
collectionName = collections[0].name;
}
await ensureElectron().setPendingUploadCollection(collectionName);
};
export const updatePendingUploads = async (files: FileWithCollection[]) => {
const filePaths = [];
for (const fileWithCollection of files) {
if (fileWithCollection.isLivePhoto) {
filePaths.push(
(fileWithCollection.livePhotoAssets.image as ElectronFile).path,
(fileWithCollection.livePhotoAssets.video as ElectronFile).path,
);
} else {
filePaths.push((fileWithCollection.file as ElectronFile).path);
}
}
await ensureElectron().setPendingUploadFiles("files", filePaths);
};
export const cancelRemainingUploads = async () => {
const electron = ensureElectron();
await electron.setPendingUploadCollection(undefined);
await electron.setPendingUploadFiles("zips", []);
await electron.setPendingUploadFiles("files", []);
};

17
web/apps/photos/src/services/upload/uploadManager.ts
View file

@ -8,13 +8,16 @@ import { Events, eventBus } from "@ente/shared/events";
import { Remote } from "comlink";
import { UPLOAD_RESULT, UPLOAD_STAGES } from "constants/upload";
import isElectron from "is-electron";
import ImportService from "services/importService";
import {
cancelRemainingUploads,
updatePendingUploads,
} from "services/pending-uploads";
import {
getLocalPublicFiles,
getPublicCollectionUID,
} from "services/publicCollectionService";
import { getDisableCFUploadProxyFlag } from "services/userService";
import watchFolderService from "services/watch";
import watcher from "services/watch";
import { Collection } from "types/collection";
import { EncryptedEnteFile, EnteFile } from "types/file";
import { SetFiles } from "types/gallery";
@ -177,7 +180,7 @@ class UploadManager {
if (e.message === CustomError.UPLOAD_CANCELLED) {
if (isElectron()) {
this.remainingFiles = [];
await ImportService.cancelRemainingUploads();
await cancelRemainingUploads();
}
} else {
log.error("uploading failed with error", e);
@ -387,13 +390,15 @@ class UploadManager {
uploadedFile: EncryptedEnteFile,
) {
if (isElectron()) {
await watchFolderService.onFileUpload(
if (watcher.isUploadRunning()) {
await watcher.onFileUpload(
fileUploadResult,
fileWithCollection,
uploadedFile,
);
}
}
}
public cancelRunningUpload() {
log.info("user cancelled running upload");
@ -431,12 +436,12 @@ class UploadManager {
this.remainingFiles = this.remainingFiles.filter(
(file) => !areFileWithCollectionsSame(file, fileWithCollection),
);
await ImportService.updatePendingUploads(this.remainingFiles);
await updatePendingUploads(this.remainingFiles);
}
}
public shouldAllowNewUpload = () => {
return !this.uploadInProgress || watchFolderService.isUploadRunning();
return !this.uploadInProgress || watcher.isUploadRunning();
};
}

887
web/apps/photos/src/services/watch.ts
View file

File diff suppressed because it is too large Load diff

7
web/apps/photos/src/types/upload/index.ts
View file

@ -156,13 +156,6 @@ export interface ParsedExtractedMetadata {
height: number;
}
// This is used to prompt the user the make upload strategy choice
export interface ImportSuggestion {
rootFolderName: string;
hasNestedFolders: boolean;
hasRootLevelFileWithFolder: boolean;
}
export interface PublicUploadProps {
token: string;
passwordToken: string;

24
web/apps/photos/src/types/watchFolder/index.ts
View file

@ -1,24 +0,0 @@
import { UPLOAD_STRATEGY } from "constants/upload";
import { ElectronFile } from "types/upload";
export interface WatchMappingSyncedFile {
path: string;
uploadedFileID: number;
collectionID: number;
}
export interface WatchMapping {
rootFolderName: string;
folderPath: string;
uploadStrategy: UPLOAD_STRATEGY;
syncedFiles: WatchMappingSyncedFile[];
ignoredFiles: string[];
}
export interface EventQueueItem {
type: "upload" | "trash";
folderPath: string;
collectionName?: string;
paths?: string[];
files?: ElectronFile[];
}

6
web/apps/photos/src/utils/storage/mlIDbStorage.ts
View file

@ -97,10 +97,8 @@ class MLIDbStorage {
wasMLSearchEnabled = searchConfig.enabled;
}
} catch (e) {
log.info(
"Ignoring likely harmless error while trying to determine ML search status during migration",
e,
);
// The configs store might not exist (e.g. during logout).
// Ignore.
}
log.info(
`Previous ML database v${oldVersion} had ML search ${wasMLSearchEnabled ? "enabled" : "disabled"}`,

6
web/apps/photos/src/utils/ui/index.tsx
View file

@ -1,5 +1,5 @@
import { ensureElectron } from "@/next/electron";
import { AppUpdateInfo } from "@/next/types/ipc";
import { AppUpdate } from "@/next/types/ipc";
import { logoutUser } from "@ente/accounts/services/user";
import { DialogBoxAttributes } from "@ente/shared/components/DialogBox/types";
import AutoAwesomeOutlinedIcon from "@mui/icons-material/AutoAwesomeOutlined";
@ -55,7 +55,7 @@ export const getTrashFileMessage = (deleteFileHelper): DialogBoxAttributes => ({
export const getUpdateReadyToInstallMessage = ({
version,
}: AppUpdateInfo): DialogBoxAttributes => ({
}: AppUpdate): DialogBoxAttributes => ({
icon: <AutoAwesomeOutlinedIcon />,
title: t("UPDATE_AVAILABLE"),
content: t("UPDATE_INSTALLABLE_MESSAGE"),
@ -73,7 +73,7 @@ export const getUpdateReadyToInstallMessage = ({
export const getUpdateAvailableForDownloadMessage = ({
version,
}: AppUpdateInfo): DialogBoxAttributes => ({
}: AppUpdate): DialogBoxAttributes => ({
icon: <AutoAwesomeOutlinedIcon />,
title: t("UPDATE_AVAILABLE"),
content: t("UPDATE_AVAILABLE_MESSAGE"),

46
web/apps/photos/src/utils/upload/index.ts
View file

@ -1,18 +1,10 @@
import { basename, dirname } from "@/next/file";
import { FILE_TYPE } from "constants/file";
import {
A_SEC_IN_MICROSECONDS,
DEFAULT_IMPORT_SUGGESTION,
PICKED_UPLOAD_TYPE,
} from "constants/upload";
import { A_SEC_IN_MICROSECONDS, PICKED_UPLOAD_TYPE } from "constants/upload";
import isElectron from "is-electron";
import { exportMetadataDirectoryName } from "services/export";
import { EnteFile } from "types/file";
import {
ElectronFile,
FileWithCollection,
ImportSuggestion,
Metadata,
} from "types/upload";
import { ElectronFile, FileWithCollection, Metadata } from "types/upload";
const TYPE_JSON = "json";
const DEDUPE_COLLECTION = new Set(["icloud library", "icloudlibrary"]);
@ -110,15 +102,36 @@ export function areFileWithCollectionsSame(
return firstFile.localID === secondFile.localID;
}
/**
* Return true if all the paths in the given list are items that belong to the
* same (arbitrary) directory.
*
* Empty list of paths is considered to be in the same directory.
*/
export const areAllInSameDirectory = (paths: string[]) =>
new Set(paths.map(dirname)).size == 1;
// This is used to prompt the user the make upload strategy choice
export interface ImportSuggestion {
rootFolderName: string;
hasNestedFolders: boolean;
hasRootLevelFileWithFolder: boolean;
}
export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = {
rootFolderName: "",
hasNestedFolders: false,
hasRootLevelFileWithFolder: false,
};
export function getImportSuggestion(
uploadType: PICKED_UPLOAD_TYPE,
toUploadFiles: File[] | ElectronFile[],
paths: string[],
): ImportSuggestion {
if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) {
return DEFAULT_IMPORT_SUGGESTION;
}
const paths: string[] = toUploadFiles.map((file) => file["path"]);
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
@ -209,3 +222,10 @@ export function filterOutSystemFiles(files: File[] | ElectronFile[]) {
export function isSystemFile(file: File | ElectronFile) {
return file.name.startsWith(".");
}
/**
* Return true if the file at the given {@link path} is hidden.
*
* Hidden files are those whose names begin with a "." (dot).
*/
export const isHiddenFile = (path: string) => basename(path).startsWith(".");

2
web/apps/photos/tests/zip-file-reading.test.ts
View file

@ -96,7 +96,7 @@ export const testZipWithRootFileReadingTest = async () => {
const importSuggestion = getImportSuggestion(
PICKED_UPLOAD_TYPE.ZIPS,
files,
files.map((file) => file["path"]),
);
if (!importSuggestion.rootFolderName) {
throw Error(

39
web/packages/next/file.ts
View file

@ -17,8 +17,12 @@ type FileNameComponents = [name: string, extension: string | undefined];
*/
export const nameAndExtension = (fileName: string): FileNameComponents => {
const i = fileName.lastIndexOf(".");
// No extension
if (i == -1) return [fileName, undefined];
else return [fileName.slice(0, i), fileName.slice(i + 1)];
// A hidden file without an extension, e.g. ".gitignore"
if (i == 0) return [fileName, undefined];
// Both components present, just omit the dot.
return [fileName.slice(0, i), fileName.slice(i + 1)];
};
/**
@ -29,6 +33,39 @@ export const nameAndExtension = (fileName: string): FileNameComponents => {
export const fileNameFromComponents = (components: FileNameComponents) =>
components.filter((x) => !!x).join(".");
/**
* Return the file name portion from the given {@link path}.
*
* This tries to emulate the UNIX `basename` command. In particular, any
* trailing slashes on the path are trimmed, so this function can be used to get
* the name of the directory too.
*
* The path is assumed to use POSIX separators ("/").
*/
export const basename = (path: string) => {
const pathComponents = path.split("/");
for (let i = pathComponents.length - 1; i >= 0; i--)
if (pathComponents[i] !== "") return pathComponents[i];
return path;
};
/**
* Return the directory portion from the given {@link path}.
*
* This tries to emulate the UNIX `dirname` command. In particular, any trailing
* slashes on the path are trimmed, so this function can be used to get the path
* leading up to a directory too.
*
* The path is assumed to use POSIX separators ("/").
*/
export const dirname = (path: string) => {
const pathComponents = path.split("/");
while (pathComponents.pop() == "") {
/* no-op */
}
return pathComponents.join("/");
};
export function getFileNameSize(file: File | ElectronFile) {
return `${file.name}_${convertBytesToHumanReadable(file.size)}`;
}

243
web/packages/next/types/ipc.ts
View file

@ -5,22 +5,6 @@
import type { ElectronFile } from "./file";
export interface AppUpdateInfo {
autoUpdatable: boolean;
version: string;
}
export enum FILE_PATH_TYPE {
FILES = "files",
ZIPS = "zips",
}
export enum PICKED_UPLOAD_TYPE {
FILES = "files",
FOLDERS = "folders",
ZIPS = "zips",
}
/**
* Extra APIs provided by our Node.js layer when our code is running inside our
* desktop (Electron) app.
@ -111,7 +95,7 @@ export interface Electron {
* Note: Setting a callback clears any previous callbacks.
*/
onAppUpdateAvailable: (
cb?: ((updateInfo: AppUpdateInfo) => void) | undefined,
cb?: ((update: AppUpdate) => void) | undefined,
) => void;
/**
@ -199,6 +183,12 @@ export interface Electron {
* @param contents The string contents to write.
*/
writeFile: (path: string, contents: string) => Promise<void>;
/**
* Return true if there is an item at {@link dirPath}, and it is as
* directory.
*/
isDir: (dirPath: string) => Promise<boolean>;
};
/*
@ -284,73 +274,211 @@ export interface Electron {
// - Watch
registerWatcherFunctions: (
addFile: (file: ElectronFile) => Promise<void>,
removeFile: (path: string) => Promise<void>,
removeFolder: (folderPath: string) => Promise<void>,
) => void;
/**
* Interface with the file system watcher running in our Node.js layer.
*
* [Note: Folder vs Directory in the context of FolderWatch-es]
*
* A note on terminology: The word "folder" is used to the top level root
* folder for which a {@link FolderWatch} has been added. This folder is
* also in 1-1 correspondence to be a directory on the user's disk. It can
* have other, nested directories too (which may or may not be getting
* mapped to separate Ente collections), but we'll not refer to these nested
* directories as folders - only the root of the tree, which the user
* dragged/dropped or selected to set up the folder watch, will be referred
* to as a folder when naming things.
*/
watch: {
/**
* Return the list of folder watches, pruning non-existing directories.
*
* The list of folder paths (and auxillary details) is persisted in the
* Node.js layer. The implementation of this function goes through the
* list, permanently removes any watches whose on-disk directory is no
* longer present, and returns this pruned list of watches.
*/
get: () => Promise<FolderWatch[]>;
addWatchMapping: (
collectionName: string,
/**
* Add a new folder watch for the given {@link folderPath}.
*
* This adds a new entry in the list of watches (persisting them on
* disk), and also starts immediately observing for file system events
* that happen within {@link folderPath}.
*
* @param collectionMapping Determines how nested directories (if any)
* get mapped to Ente collections.
*
* @returns The updated list of watches.
*/
add: (
folderPath: string,
collectionMapping: CollectionMapping,
) => Promise<FolderWatch[]>;
/**
* Remove the pre-existing watch for the given {@link folderPath}.
*
* Persist this removal, and also stop listening for file system events
* that happen within the {@link folderPath}.
*
* @returns The updated list of watches.
*/
remove: (folderPath: string) => Promise<FolderWatch[]>;
/**
* Update the list of synced files for the folder watch associated
* with the given {@link folderPath}.
*/
updateSyncedFiles: (
syncedFiles: FolderWatch["syncedFiles"],
folderPath: string,
uploadStrategy: number,
) => Promise<void>;
removeWatchMapping: (folderPath: string) => Promise<void>;
getWatchMappings: () => Promise<FolderWatch[]>;
updateWatchMappingSyncedFiles: (
/**
* Update the list of ignored file paths for the folder watch
* associated with the given {@link folderPath}.
*/
updateIgnoredFiles: (
ignoredFiles: FolderWatch["ignoredFiles"],
folderPath: string,
files: FolderWatch["syncedFiles"],
) => Promise<void>;
updateWatchMappingIgnoredFiles: (
folderPath: string,
files: FolderWatch["ignoredFiles"],
) => Promise<void>;
/**
* Register the function to invoke when a file is added in one of the
* folders we are watching.
*
* The callback function is passed the path to the file that was added,
* and the folder watch it was associated with.
*
* The path is guaranteed to use POSIX separators ('/').
*/
onAddFile: (f: (path: string, watch: FolderWatch) => void) => void;
// - FS legacy
isFolder: (dirPath: string) => Promise<boolean>;
/**
* Register the function to invoke when a file is removed in one of the
* folders we are watching.
*
* The callback function is passed the path to the file that was
* removed, and the folder watch it was associated with.
*
* The path is guaranteed to use POSIX separators ('/').
*/
onRemoveFile: (f: (path: string, watch: FolderWatch) => void) => void;
/**
* Register the function to invoke when a directory is removed in one of
* the folders we are watching.
*
* The callback function is passed the path to the directory that was
* removed, and the folder watch it was associated with.
*
* The path is guaranteed to use POSIX separators ('/').
*/
onRemoveDir: (f: (path: string, watch: FolderWatch) => void) => void;
/**
* Return the paths of all the files under the given folder.
*
* This function walks the directory tree starting at {@link folderPath}
* and returns a list of the absolute paths of all the files that exist
* therein. It will recursively traverse into nested directories, and
* return the absolute paths of the files there too.
*
* The returned paths are guaranteed to use POSIX separators ('/').
*/
findFiles: (folderPath: string) => Promise<string[]>;
};
// - Upload
getPendingUploads: () => Promise<{
files: ElectronFile[];
collectionName: string;
type: string;
}>;
setToUploadFiles: (
/** TODO(MR): This is the actual type */
// type: FILE_PATH_TYPE,
type: PICKED_UPLOAD_TYPE,
/**
* Return any pending uploads that were previously enqueued but haven't yet
* been completed.
*
* The state of pending uploads is persisted in the Node.js layer.
*
* Note that we might have both outstanding zip and regular file uploads at
* the same time. In such cases, the zip file ones get precedence.
*/
pendingUploads: () => Promise<PendingUploads | undefined>;
/**
* Set or clear the name of the collection where the pending upload is
* directed to.
*/
setPendingUploadCollection: (collectionName: string) => Promise<void>;
/**
* Update the list of files (of {@link type}) associated with the pending
* upload.
*/
setPendingUploadFiles: (
type: PendingUploads["type"],
filePaths: string[],
) => Promise<void>;
// -
getElectronFilesFromGoogleZip: (
filePath: string,
) => Promise<ElectronFile[]>;
setToUploadCollection: (collectionName: string) => Promise<void>;
getDirFiles: (dirPath: string) => Promise<ElectronFile[]>;
}
/**
* Data passed across the IPC bridge when an app update is available.
*/
export interface AppUpdate {
/** `true` if the user automatically update to this (new) version */
autoUpdatable: boolean;
/** The new version that is available */
version: string;
}
/**
* A top level folder that was selected by the user for watching.
*
* The user can set up multiple such watches. Each of these can in turn be
* syncing multiple on disk folders to one or more (dependening on the
* {@link uploadStrategy}) Ente albums.
* syncing multiple on disk folders to one or more Ente collections (depending
* on the value of {@link collectionMapping}).
*
* This type is passed across the IPC boundary. It is persisted on the Node.js
* side.
*/
export interface FolderWatch {
rootFolderName: string;
uploadStrategy: number;
/**
* Specify if nested files should all be mapped to the same single root
* collection, or if there should be a collection per directory that has
* files. @see {@link CollectionMapping}.
*/
collectionMapping: CollectionMapping;
/**
* The path to the (root) folder we are watching.
*/
folderPath: string;
/**
* Files that have already been uploaded.
*/
syncedFiles: FolderWatchSyncedFile[];
/**
* Files (paths) that should be ignored when uploading.
*/
ignoredFiles: string[];
}
/**
* The ways in which directories are mapped to collection.
*
* This comes into play when we have nested directories that we are trying to
* upload or watch on the user's local file system.
*/
export type CollectionMapping =
/** All files go into a single collection named after the root directory. */
| "root"
/** Each file goes to a collection named after its parent directory. */
| "parent";
/**
* An on-disk file that was synced as part of a folder watch.
*/
@ -359,3 +487,16 @@ export interface FolderWatchSyncedFile {
uploadedFileID: number;
collectionID: number;
}
/**
* When the user starts an upload, we remember the files they'd selected or drag
* and dropped so that we can resume (if needed) when the app restarts after
* being stopped in the middle of the uploads.
*/
export interface PendingUploads {
/** The collection to which we're uploading */
collectionName: string;
/* The upload can be either of a Google Takeout zip, or regular files */
type: "files" | "zips";
files: ElectronFile[];
}

7
web/packages/utils/ensure.ts Normal file
View file

@ -0,0 +1,7 @@
/**
* Throw an exception if the given value is undefined.
*/
export const ensure = <T>(v: T | undefined): T => {
if (v === undefined) throw new Error("Required value was not found");
return v;
};