diff --git a/desktop/src/main.ts b/desktop/src/main.ts index 8526e2363..467d9c881 100644 --- a/desktop/src/main.ts +++ b/desktop/src/main.ts @@ -24,9 +24,10 @@ import { attachFSWatchIPCHandlers, attachIPCHandlers } from "./main/ipc"; import log, { initLogging } from "./main/log"; import { createApplicationMenu, createTrayContextMenu } from "./main/menu"; import { setupAutoUpdater } from "./main/services/app-update"; -import autoLauncher from "./main/services/autoLauncher"; -import { initWatcher } from "./main/services/chokidar"; +import autoLauncher from "./main/services/auto-launcher"; +import { createWatcher } from "./main/services/watch"; import { userPreferences } from "./main/stores/user-preferences"; +import { migrateLegacyWatchStoreIfNeeded } from "./main/stores/watch"; import { registerStreamProtocol } from "./main/stream"; import { isDev } from "./main/util"; @@ -196,8 +197,6 @@ const createMainWindow = async () => { window.maximize(); } - window.loadURL(rendererURL); - // Open the DevTools automatically when running in dev mode if (isDev) window.webContents.openDevTools(); @@ -296,6 +295,21 @@ const deleteLegacyDiskCacheDirIfExists = async () => { } }; +/** + * Older versions of our app used to keep a keys.json. It is not needed anymore, + * remove it if it exists. + * + * This code was added March 2024, and can be removed after some time once most + * people have upgraded to newer versions. + */ +const deleteLegacyKeysStoreIfExists = async () => { + const keysStore = path.join(app.getPath("userData"), "keys.json"); + if (existsSync(keysStore)) { + log.info(`Removing legacy keys store at ${keysStore}`); + await fs.rm(keysStore); + } +}; + const main = () => { const gotTheLock = app.requestSingleInstanceLock(); if (!gotTheLock) { @@ -311,6 +325,7 @@ const main = () => { setupRendererServer(); registerPrivilegedSchemes(); increaseDiskCache(); + migrateLegacyWatchStoreIfNeeded(); app.on("second-instance", () => { // Someone tried to run a second instance, we should focus our window. @@ -325,19 +340,26 @@ const main = () => { // // Note that some Electron APIs can only be used after this event occurs. app.on("ready", async () => { + // Create window and prepare for renderer mainWindow = await createMainWindow(); - Menu.setApplicationMenu(await createApplicationMenu(mainWindow)); - setupTrayItem(mainWindow); attachIPCHandlers(); - attachFSWatchIPCHandlers(initWatcher(mainWindow)); + attachFSWatchIPCHandlers(createWatcher(mainWindow)); registerStreamProtocol(); - if (!isDev) setupAutoUpdater(mainWindow); handleDownloads(mainWindow); handleExternalLinks(mainWindow); addAllowOriginHeader(mainWindow); + // Start loading the renderer + mainWindow.loadURL(rendererURL); + + // Continue on with the rest of the startup sequence + Menu.setApplicationMenu(await createApplicationMenu(mainWindow)); + setupTrayItem(mainWindow); + if (!isDev) setupAutoUpdater(mainWindow); + try { deleteLegacyDiskCacheDirIfExists(); + deleteLegacyKeysStoreIfExists(); } catch (e) { // Log but otherwise ignore errors during non-critical startup // actions. diff --git a/desktop/src/main/fs.ts b/desktop/src/main/fs.ts index 36de710c3..2428d3a80 100644 --- a/desktop/src/main/fs.ts +++ b/desktop/src/main/fs.ts @@ -22,10 +22,8 @@ export const fsReadTextFile = async (filePath: string) => export const fsWriteFile = (path: string, contents: string) => fs.writeFile(path, contents); -/* TODO: Audit below this */ - -export const isFolder = async (dirPath: string) => { +export const fsIsDir = async (dirPath: string) => { if (!existsSync(dirPath)) return false; - const stats = await fs.stat(dirPath); - return stats.isDirectory(); + const stat = await fs.stat(dirPath); + return stat.isDirectory(); }; diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index a5de4514f..eab2e8b59 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -10,7 +10,12 @@ import type { FSWatcher } from "chokidar"; import { ipcMain } from "electron/main"; -import type { ElectronFile, FILE_PATH_TYPE, FolderWatch } from "../types/ipc"; +import type { + CollectionMapping, + ElectronFile, + FolderWatch, + PendingUploads, +} from "../types/ipc"; import { selectDirectory, showUploadDirsDialog, @@ -19,13 +24,13 @@ import { } from "./dialogs"; import { fsExists, + fsIsDir, fsMkdirIfNeeded, fsReadTextFile, fsRename, fsRm, fsRmdir, fsWriteFile, - isFolder, } from "./fs"; import { logToDisk } from "./log"; import { @@ -49,16 +54,17 @@ import { } from "./services/store"; import { getElectronFilesFromGoogleZip, - getPendingUploads, - setToUploadCollection, - setToUploadFiles, + pendingUploads, + setPendingUploadCollection, + setPendingUploadFiles, } from "./services/upload"; import { - addWatchMapping, - getWatchMappings, - removeWatchMapping, - updateWatchMappingIgnoredFiles, - updateWatchMappingSyncedFiles, + watchAdd, + watchFindFiles, + watchGet, + watchRemove, + watchUpdateIgnoredFiles, + watchUpdateSyncedFiles, } from "./services/watch"; import { openDirectory, openLogDirectory } from "./util"; @@ -132,6 +138,8 @@ export const attachIPCHandlers = () => { fsWriteFile(path, contents), ); + ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath)); + // - Conversion ipcMain.handle("convertToJPEG", (_, fileData, filename) => @@ -183,28 +191,26 @@ export const attachIPCHandlers = () => { ipcMain.handle("showUploadZipDialog", () => showUploadZipDialog()); - // - FS Legacy - - ipcMain.handle("isFolder", (_, dirPath: string) => isFolder(dirPath)); - // - Upload - ipcMain.handle("getPendingUploads", () => getPendingUploads()); + ipcMain.handle("pendingUploads", () => pendingUploads()); + + ipcMain.handle("setPendingUploadCollection", (_, collectionName: string) => + setPendingUploadCollection(collectionName), + ); ipcMain.handle( - "setToUploadFiles", - (_, type: FILE_PATH_TYPE, filePaths: string[]) => - setToUploadFiles(type, filePaths), + "setPendingUploadFiles", + (_, type: PendingUploads["type"], filePaths: string[]) => + setPendingUploadFiles(type, filePaths), ); + // - + ipcMain.handle("getElectronFilesFromGoogleZip", (_, filePath: string) => getElectronFilesFromGoogleZip(filePath), ); - ipcMain.handle("setToUploadCollection", (_, collectionName: string) => - setToUploadCollection(collectionName), - ); - ipcMain.handle("getDirFiles", (_, dirPath: string) => getDirFiles(dirPath)); }; @@ -213,42 +219,36 @@ export const attachIPCHandlers = () => { * watch folder functionality. * * It gets passed a {@link FSWatcher} instance which it can then forward to the - * actual handlers. + * actual handlers if they need access to it to do their thing. */ export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => { // - Watch - ipcMain.handle( - "addWatchMapping", - ( - _, - collectionName: string, - folderPath: string, - uploadStrategy: number, - ) => - addWatchMapping( - watcher, - collectionName, - folderPath, - uploadStrategy, - ), - ); - - ipcMain.handle("removeWatchMapping", (_, folderPath: string) => - removeWatchMapping(watcher, folderPath), - ); - - ipcMain.handle("getWatchMappings", () => getWatchMappings()); + ipcMain.handle("watchGet", () => watchGet(watcher)); ipcMain.handle( - "updateWatchMappingSyncedFiles", - (_, folderPath: string, files: FolderWatch["syncedFiles"]) => - updateWatchMappingSyncedFiles(folderPath, files), + "watchAdd", + (_, folderPath: string, collectionMapping: CollectionMapping) => + watchAdd(watcher, folderPath, collectionMapping), + ); + + ipcMain.handle("watchRemove", (_, folderPath: string) => + watchRemove(watcher, folderPath), ); ipcMain.handle( - "updateWatchMappingIgnoredFiles", - (_, folderPath: string, files: FolderWatch["ignoredFiles"]) => - updateWatchMappingIgnoredFiles(folderPath, files), + "watchUpdateSyncedFiles", + (_, syncedFiles: FolderWatch["syncedFiles"], folderPath: string) => + watchUpdateSyncedFiles(syncedFiles, folderPath), + ); + + ipcMain.handle( + "watchUpdateIgnoredFiles", + (_, ignoredFiles: FolderWatch["ignoredFiles"], folderPath: string) => + watchUpdateIgnoredFiles(ignoredFiles, folderPath), + ); + + ipcMain.handle("watchFindFiles", (_, folderPath: string) => + watchFindFiles(folderPath), ); }; diff --git a/desktop/src/main/menu.ts b/desktop/src/main/menu.ts index 3441f3f2a..bd8810428 100644 --- a/desktop/src/main/menu.ts +++ b/desktop/src/main/menu.ts @@ -7,7 +7,7 @@ import { } from "electron"; import { allowWindowClose } from "../main"; import { forceCheckForAppUpdates } from "./services/app-update"; -import autoLauncher from "./services/autoLauncher"; +import autoLauncher from "./services/auto-launcher"; import { userPreferences } from "./stores/user-preferences"; import { openLogDirectory } from "./util"; diff --git a/desktop/src/main/platform.ts b/desktop/src/main/platform.ts deleted file mode 100644 index 1c3bb4584..000000000 --- a/desktop/src/main/platform.ts +++ /dev/null @@ -1,19 +0,0 @@ -export function isPlatform(platform: "mac" | "windows" | "linux") { - return getPlatform() === platform; -} - -export function getPlatform(): "mac" | "windows" | "linux" { - switch (process.platform) { - case "aix": - case "freebsd": - case "linux": - case "openbsd": - case "android": - return "linux"; - case "darwin": - case "sunos": - return "mac"; - case "win32": - return "windows"; - } -} diff --git a/desktop/src/main/services/app-update.ts b/desktop/src/main/services/app-update.ts index b47448501..a3f4d3bed 100644 --- a/desktop/src/main/services/app-update.ts +++ b/desktop/src/main/services/app-update.ts @@ -1,9 +1,9 @@ import { compareVersions } from "compare-versions"; -import { app, BrowserWindow } from "electron"; import { default as electronLog } from "electron-log"; import { autoUpdater } from "electron-updater"; +import { app, BrowserWindow } from "electron/main"; import { allowWindowClose } from "../../main"; -import { AppUpdateInfo } from "../../types/ipc"; +import { AppUpdate } from "../../types/ipc"; import log from "../log"; import { userPreferences } from "../stores/user-preferences"; @@ -52,8 +52,8 @@ const checkForUpdatesAndNotify = async (mainWindow: BrowserWindow) => { return; } - const showUpdateDialog = (updateInfo: AppUpdateInfo) => - mainWindow.webContents.send("appUpdateAvailable", updateInfo); + const showUpdateDialog = (update: AppUpdate) => + mainWindow.webContents.send("appUpdateAvailable", update); log.debug(() => "Attempting auto update"); autoUpdater.downloadUpdate(); diff --git a/desktop/src/main/services/auto-launcher.ts b/desktop/src/main/services/auto-launcher.ts new file mode 100644 index 000000000..c704f7399 --- /dev/null +++ b/desktop/src/main/services/auto-launcher.ts @@ -0,0 +1,51 @@ +import AutoLaunch from "auto-launch"; +import { app } from "electron/main"; + +class AutoLauncher { + /** + * This property will be set and used on Linux and Windows. On macOS, + * there's a separate API + */ + private autoLaunch?: AutoLaunch; + + constructor() { + if (process.platform != "darwin") { + this.autoLaunch = new AutoLaunch({ + name: "ente", + isHidden: true, + }); + } + } + + async isEnabled() { + const autoLaunch = this.autoLaunch; + if (autoLaunch) { + return await autoLaunch.isEnabled(); + } else { + return app.getLoginItemSettings().openAtLogin; + } + } + + async toggleAutoLaunch() { + const isEnabled = await this.isEnabled(); + const autoLaunch = this.autoLaunch; + if (autoLaunch) { + if (isEnabled) await autoLaunch.disable(); + else await autoLaunch.enable(); + } else { + if (isEnabled) app.setLoginItemSettings({ openAtLogin: false }); + else app.setLoginItemSettings({ openAtLogin: true }); + } + } + + async wasAutoLaunched() { + if (this.autoLaunch) { + return app.commandLine.hasSwitch("hidden"); + } else { + // TODO(MR): This apparently doesn't work anymore. + return app.getLoginItemSettings().wasOpenedAtLogin; + } + } +} + +export default new AutoLauncher(); diff --git a/desktop/src/main/services/autoLauncher.ts b/desktop/src/main/services/autoLauncher.ts deleted file mode 100644 index 614c151ba..000000000 --- a/desktop/src/main/services/autoLauncher.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { AutoLauncherClient } from "../../types/main"; -import { isPlatform } from "../platform"; -import linuxAndWinAutoLauncher from "./autoLauncherClients/linuxAndWinAutoLauncher"; -import macAutoLauncher from "./autoLauncherClients/macAutoLauncher"; - -class AutoLauncher { - private client: AutoLauncherClient; - async init() { - if (isPlatform("linux") || isPlatform("windows")) { - this.client = linuxAndWinAutoLauncher; - } else { - this.client = macAutoLauncher; - } - // migrate old auto launch settings for windows from mac auto launcher to linux and windows auto launcher - if (isPlatform("windows") && (await macAutoLauncher.isEnabled())) { - await macAutoLauncher.toggleAutoLaunch(); - await linuxAndWinAutoLauncher.toggleAutoLaunch(); - } - } - async isEnabled() { - if (!this.client) { - await this.init(); - } - return await this.client.isEnabled(); - } - async toggleAutoLaunch() { - if (!this.client) { - await this.init(); - } - await this.client.toggleAutoLaunch(); - } - - async wasAutoLaunched() { - if (!this.client) { - await this.init(); - } - return this.client.wasAutoLaunched(); - } -} - -export default new AutoLauncher(); diff --git a/desktop/src/main/services/autoLauncherClients/linuxAndWinAutoLauncher.ts b/desktop/src/main/services/autoLauncherClients/linuxAndWinAutoLauncher.ts deleted file mode 100644 index 0d2c1bb42..000000000 --- a/desktop/src/main/services/autoLauncherClients/linuxAndWinAutoLauncher.ts +++ /dev/null @@ -1,39 +0,0 @@ -import AutoLaunch from "auto-launch"; -import { app } from "electron"; -import { AutoLauncherClient } from "../../../types/main"; - -const LAUNCHED_AS_HIDDEN_FLAG = "hidden"; - -class LinuxAndWinAutoLauncher implements AutoLauncherClient { - private instance: AutoLaunch; - constructor() { - const autoLauncher = new AutoLaunch({ - name: "ente", - isHidden: true, - }); - this.instance = autoLauncher; - } - async isEnabled() { - return await this.instance.isEnabled(); - } - async toggleAutoLaunch() { - if (await this.isEnabled()) { - await this.disableAutoLaunch(); - } else { - await this.enableAutoLaunch(); - } - } - - async wasAutoLaunched() { - return app.commandLine.hasSwitch(LAUNCHED_AS_HIDDEN_FLAG); - } - - private async disableAutoLaunch() { - await this.instance.disable(); - } - private async enableAutoLaunch() { - await this.instance.enable(); - } -} - -export default new LinuxAndWinAutoLauncher(); diff --git a/desktop/src/main/services/autoLauncherClients/macAutoLauncher.ts b/desktop/src/main/services/autoLauncherClients/macAutoLauncher.ts deleted file mode 100644 index 00320e870..000000000 --- a/desktop/src/main/services/autoLauncherClients/macAutoLauncher.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { app } from "electron"; -import { AutoLauncherClient } from "../../../types/main"; - -class MacAutoLauncher implements AutoLauncherClient { - async isEnabled() { - return app.getLoginItemSettings().openAtLogin; - } - async toggleAutoLaunch() { - if (await this.isEnabled()) { - this.disableAutoLaunch(); - } else { - this.enableAutoLaunch(); - } - } - - async wasAutoLaunched() { - return app.getLoginItemSettings().wasOpenedAtLogin; - } - - private disableAutoLaunch() { - app.setLoginItemSettings({ openAtLogin: false }); - } - private enableAutoLaunch() { - app.setLoginItemSettings({ openAtLogin: true }); - } -} - -export default new MacAutoLauncher(); diff --git a/desktop/src/main/services/chokidar.ts b/desktop/src/main/services/chokidar.ts deleted file mode 100644 index 5d7284d2a..000000000 --- a/desktop/src/main/services/chokidar.ts +++ /dev/null @@ -1,45 +0,0 @@ -import chokidar from "chokidar"; -import { BrowserWindow } from "electron"; -import path from "path"; -import log from "../log"; -import { getElectronFile } from "./fs"; -import { getWatchMappings } from "./watch"; - -/** - * Convert a file system {@link filePath} that uses the local system specific - * path separators into a path that uses POSIX file separators. - */ -const normalizeToPOSIX = (filePath: string) => - filePath.split(path.sep).join(path.posix.sep); - -export function initWatcher(mainWindow: BrowserWindow) { - const mappings = getWatchMappings(); - const folderPaths = mappings.map((mapping) => { - return mapping.folderPath; - }); - - const watcher = chokidar.watch(folderPaths, { - awaitWriteFinish: true, - }); - watcher - .on("add", async (path) => { - mainWindow.webContents.send( - "watch-add", - await getElectronFile(normalizeToPOSIX(path)), - ); - }) - .on("unlink", (path) => { - mainWindow.webContents.send("watch-unlink", normalizeToPOSIX(path)); - }) - .on("unlinkDir", (path) => { - mainWindow.webContents.send( - "watch-unlink-dir", - normalizeToPOSIX(path), - ); - }) - .on("error", (error) => { - log.error("Error while watching files", error); - }); - - return watcher; -} diff --git a/desktop/src/main/services/fs.ts b/desktop/src/main/services/fs.ts index 7a29d581b..30ccf146b 100644 --- a/desktop/src/main/services/fs.ts +++ b/desktop/src/main/services/fs.ts @@ -91,19 +91,6 @@ export async function getElectronFile(filePath: string): Promise { }; } -export const getValidPaths = (paths: string[]) => { - if (!paths) { - return [] as string[]; - } - return paths.filter(async (path) => { - try { - await fs.stat(path).then((stat) => stat.isFile()); - } catch (e) { - return false; - } - }); -}; - export const getZipFileStream = async ( zip: StreamZip.StreamZipAsync, filePath: string, diff --git a/desktop/src/main/services/imageProcessor.ts b/desktop/src/main/services/imageProcessor.ts index e1119d50f..f636c153a 100644 --- a/desktop/src/main/services/imageProcessor.ts +++ b/desktop/src/main/services/imageProcessor.ts @@ -3,7 +3,6 @@ import fs from "node:fs/promises"; import path from "path"; import { CustomErrors, ElectronFile } from "../../types/ipc"; import log from "../log"; -import { isPlatform } from "../platform"; import { writeStream } from "../stream"; import { generateTempFilePath } from "../temp"; import { execAsync, isDev } from "../util"; @@ -74,9 +73,8 @@ export async function convertToJPEG( fileData: Uint8Array, filename: string, ): Promise { - if (isPlatform("windows")) { + if (process.platform == "win32") throw Error(CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED); - } const convertedFileData = await convertToJPEG_(fileData, filename); return convertedFileData; } @@ -123,7 +121,7 @@ function constructConvertCommand( tempOutputFilePath: string, ) { let convertCmd: string[]; - if (isPlatform("mac")) { + if (process.platform == "darwin") { convertCmd = SIPS_HEIC_CONVERT_COMMAND_TEMPLATE.map((cmdPart) => { if (cmdPart === INPUT_PATH_PLACEHOLDER) { return tempInputFilePath; @@ -133,7 +131,7 @@ function constructConvertCommand( } return cmdPart; }); - } else if (isPlatform("linux")) { + } else if (process.platform == "linux") { convertCmd = IMAGEMAGICK_HEIC_CONVERT_COMMAND_TEMPLATE.map( (cmdPart) => { if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) { @@ -162,11 +160,10 @@ export async function generateImageThumbnail( let inputFilePath = null; let createdTempInputFile = null; try { - if (isPlatform("windows")) { + if (process.platform == "win32") throw Error( CustomErrors.WINDOWS_NATIVE_IMAGE_PROCESSING_NOT_SUPPORTED, ); - } if (!existsSync(inputFile.path)) { const tempFilePath = await generateTempFilePath(inputFile.name); await writeStream(tempFilePath, await inputFile.stream()); @@ -237,7 +234,7 @@ function constructThumbnailGenerationCommand( quality: number, ) { let thumbnailGenerationCmd: string[]; - if (isPlatform("mac")) { + if (process.platform == "darwin") { thumbnailGenerationCmd = SIPS_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map( (cmdPart) => { if (cmdPart === INPUT_PATH_PLACEHOLDER) { @@ -255,7 +252,7 @@ function constructThumbnailGenerationCommand( return cmdPart; }, ); - } else if (isPlatform("linux")) { + } else if (process.platform == "linux") { thumbnailGenerationCmd = IMAGE_MAGICK_THUMBNAIL_GENERATE_COMMAND_TEMPLATE.map((cmdPart) => { if (cmdPart === IMAGE_MAGICK_PLACEHOLDER) { diff --git a/desktop/src/main/services/store.ts b/desktop/src/main/services/store.ts index a484080f5..9ec65c8c3 100644 --- a/desktop/src/main/services/store.ts +++ b/desktop/src/main/services/store.ts @@ -1,12 +1,15 @@ import { safeStorage } from "electron/main"; -import { keysStore } from "../stores/keys.store"; -import { safeStorageStore } from "../stores/safeStorage.store"; -import { uploadStatusStore } from "../stores/upload.store"; -import { watchStore } from "../stores/watch.store"; +import { safeStorageStore } from "../stores/safe-storage"; +import { uploadStatusStore } from "../stores/upload-status"; +import { watchStore } from "../stores/watch"; +/** + * Clear all stores except user preferences. + * + * This is useful to reset state when the user logs out. + */ export const clearStores = () => { uploadStatusStore.clear(); - keysStore.clear(); safeStorageStore.clear(); watchStore.clear(); }; diff --git a/desktop/src/main/services/upload.ts b/desktop/src/main/services/upload.ts index e3fbc16e6..88c2d88d1 100644 --- a/desktop/src/main/services/upload.ts +++ b/desktop/src/main/services/upload.ts @@ -1,19 +1,23 @@ import StreamZip from "node-stream-zip"; +import { existsSync } from "original-fs"; import path from "path"; -import { ElectronFile, FILE_PATH_TYPE } from "../../types/ipc"; -import { FILE_PATH_KEYS } from "../../types/main"; -import { uploadStatusStore } from "../stores/upload.store"; -import { getElectronFile, getValidPaths, getZipFileStream } from "./fs"; +import { ElectronFile, type PendingUploads } from "../../types/ipc"; +import { + uploadStatusStore, + type UploadStatusStore, +} from "../stores/upload-status"; +import { getElectronFile, getZipFileStream } from "./fs"; -export const getPendingUploads = async () => { - const filePaths = getSavedFilePaths(FILE_PATH_TYPE.FILES); - const zipPaths = getSavedFilePaths(FILE_PATH_TYPE.ZIPS); +export const pendingUploads = async () => { const collectionName = uploadStatusStore.get("collectionName"); + const filePaths = validSavedPaths("files"); + const zipPaths = validSavedPaths("zips"); let files: ElectronFile[] = []; - let type: FILE_PATH_TYPE; + let type: PendingUploads["type"]; + if (zipPaths.length) { - type = FILE_PATH_TYPE.ZIPS; + type = "zips"; for (const zipPath of zipPaths) { files = [ ...files, @@ -23,9 +27,10 @@ export const getPendingUploads = async () => { const pendingFilePaths = new Set(filePaths); files = files.filter((file) => pendingFilePaths.has(file.path)); } else if (filePaths.length) { - type = FILE_PATH_TYPE.FILES; + type = "files"; files = await Promise.all(filePaths.map(getElectronFile)); } + return { files, collectionName, @@ -33,16 +38,56 @@ export const getPendingUploads = async () => { }; }; -export const getSavedFilePaths = (type: FILE_PATH_TYPE) => { - const paths = - getValidPaths( - uploadStatusStore.get(FILE_PATH_KEYS[type]) as string[], - ) ?? []; - - setToUploadFiles(type, paths); +export const validSavedPaths = (type: PendingUploads["type"]) => { + const key = storeKey(type); + const savedPaths = (uploadStatusStore.get(key) as string[]) ?? []; + const paths = savedPaths.filter((p) => existsSync(p)); + uploadStatusStore.set(key, paths); return paths; }; +export const setPendingUploadCollection = (collectionName: string) => { + if (collectionName) uploadStatusStore.set("collectionName", collectionName); + else uploadStatusStore.delete("collectionName"); +}; + +export const setPendingUploadFiles = ( + type: PendingUploads["type"], + filePaths: string[], +) => { + const key = storeKey(type); + if (filePaths) uploadStatusStore.set(key, filePaths); + else uploadStatusStore.delete(key); +}; + +const storeKey = (type: PendingUploads["type"]): keyof UploadStatusStore => { + switch (type) { + case "zips": + return "zipPaths"; + case "files": + return "filePaths"; + } +}; + +export const getElectronFilesFromGoogleZip = async (filePath: string) => { + const zip = new StreamZip.async({ + file: filePath, + }); + const zipName = path.basename(filePath, ".zip"); + + const entries = await zip.entries(); + const files: ElectronFile[] = []; + + for (const entry of Object.values(entries)) { + const basename = path.basename(entry.name); + if (entry.isFile && basename.length > 0 && basename[0] !== ".") { + files.push(await getZipEntryAsElectronFile(zipName, zip, entry)); + } + } + + return files; +}; + export async function getZipEntryAsElectronFile( zipName: string, zip: StreamZip.StreamZipAsync, @@ -69,39 +114,3 @@ export async function getZipEntryAsElectronFile( }, }; } - -export const setToUploadFiles = (type: FILE_PATH_TYPE, filePaths: string[]) => { - const key = FILE_PATH_KEYS[type]; - if (filePaths) { - uploadStatusStore.set(key, filePaths); - } else { - uploadStatusStore.delete(key); - } -}; - -export const setToUploadCollection = (collectionName: string) => { - if (collectionName) { - uploadStatusStore.set("collectionName", collectionName); - } else { - uploadStatusStore.delete("collectionName"); - } -}; - -export const getElectronFilesFromGoogleZip = async (filePath: string) => { - const zip = new StreamZip.async({ - file: filePath, - }); - const zipName = path.basename(filePath, ".zip"); - - const entries = await zip.entries(); - const files: ElectronFile[] = []; - - for (const entry of Object.values(entries)) { - const basename = path.basename(entry.name); - if (entry.isFile && basename.length > 0 && basename[0] !== ".") { - files.push(await getZipEntryAsElectronFile(zipName, zip, entry)); - } - } - - return files; -}; diff --git a/desktop/src/main/services/watch.ts b/desktop/src/main/services/watch.ts index 1d466d415..73a13c545 100644 --- a/desktop/src/main/services/watch.ts +++ b/desktop/src/main/services/watch.ts @@ -1,101 +1,159 @@ -import type { FSWatcher } from "chokidar"; -import ElectronLog from "electron-log"; -import { FolderWatch, WatchStoreType } from "../../types/ipc"; -import { watchStore } from "../stores/watch.store"; +import chokidar, { type FSWatcher } from "chokidar"; +import { BrowserWindow } from "electron/main"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { FolderWatch, type CollectionMapping } from "../../types/ipc"; +import { fsIsDir } from "../fs"; +import log from "../log"; +import { watchStore } from "../stores/watch"; -export const addWatchMapping = async ( - watcher: FSWatcher, - rootFolderName: string, - folderPath: string, - uploadStrategy: number, -) => { - ElectronLog.log(`Adding watch mapping: ${folderPath}`); - const watchMappings = getWatchMappings(); - if (isMappingPresent(watchMappings, folderPath)) { - throw new Error(`Watch mapping already exists`); +/** + * Create and return a new file system watcher. + * + * Internally this uses the watcher from the chokidar package. + * + * @param mainWindow The window handle is used to notify the renderer process of + * pertinent file system events. + */ +export const createWatcher = (mainWindow: BrowserWindow) => { + const send = (eventName: string) => (path: string) => + mainWindow.webContents.send(eventName, ...eventData(path)); + + const folderPaths = folderWatches().map((watch) => watch.folderPath); + + const watcher = chokidar.watch(folderPaths, { + awaitWriteFinish: true, + }); + + watcher + .on("add", send("watchAddFile")) + .on("unlink", send("watchRemoveFile")) + .on("unlinkDir", send("watchRemoveDir")) + .on("error", (error) => log.error("Error while watching files", error)); + + return watcher; +}; + +const eventData = (path: string): [string, FolderWatch] => { + path = posixPath(path); + + const watch = folderWatches().find((watch) => + path.startsWith(watch.folderPath + "/"), + ); + + if (!watch) throw new Error(`No folder watch was found for path ${path}`); + + return [path, watch]; +}; + +/** + * Convert a file system {@link filePath} that uses the local system specific + * path separators into a path that uses POSIX file separators. + */ +const posixPath = (filePath: string) => + filePath.split(path.sep).join(path.posix.sep); + +export const watchGet = (watcher: FSWatcher) => { + const [valid, deleted] = folderWatches().reduce( + ([valid, deleted], watch) => { + (fsIsDir(watch.folderPath) ? valid : deleted).push(watch); + return [valid, deleted]; + }, + [[], []], + ); + if (deleted.length) { + for (const watch of deleted) watchRemove(watcher, watch.folderPath); + setFolderWatches(valid); } + return valid; +}; - watcher.add(folderPath); +const folderWatches = (): FolderWatch[] => watchStore.get("mappings") ?? []; - watchMappings.push({ - rootFolderName, - uploadStrategy, +const setFolderWatches = (watches: FolderWatch[]) => + watchStore.set("mappings", watches); + +export const watchAdd = async ( + watcher: FSWatcher, + folderPath: string, + collectionMapping: CollectionMapping, +) => { + const watches = folderWatches(); + + if (!fsIsDir(folderPath)) + throw new Error( + `Attempting to add a folder watch for a folder path ${folderPath} that is not an existing directory`, + ); + + if (watches.find((watch) => watch.folderPath == folderPath)) + throw new Error( + `A folder watch with the given folder path ${folderPath} already exists`, + ); + + watches.push({ folderPath, + collectionMapping, syncedFiles: [], ignoredFiles: [], }); - setWatchMappings(watchMappings); + setFolderWatches(watches); + + watcher.add(folderPath); + + return watches; }; -function isMappingPresent(watchMappings: FolderWatch[], folderPath: string) { - const watchMapping = watchMappings?.find( - (mapping) => mapping.folderPath === folderPath, - ); - return !!watchMapping; -} +export const watchRemove = async (watcher: FSWatcher, folderPath: string) => { + const watches = folderWatches(); + const filtered = watches.filter((watch) => watch.folderPath != folderPath); + if (watches.length == filtered.length) + throw new Error( + `Attempting to remove a non-existing folder watch for folder path ${folderPath}`, + ); + setFolderWatches(filtered); + watcher.unwatch(folderPath); + return filtered; +}; -export const removeWatchMapping = async ( - watcher: FSWatcher, +export const watchUpdateSyncedFiles = ( + syncedFiles: FolderWatch["syncedFiles"], folderPath: string, ) => { - let watchMappings = getWatchMappings(); - const watchMapping = watchMappings.find( - (mapping) => mapping.folderPath === folderPath, + setFolderWatches( + folderWatches().map((watch) => { + if (watch.folderPath == folderPath) { + watch.syncedFiles = syncedFiles; + } + return watch; + }), ); - - if (!watchMapping) { - throw new Error(`Watch mapping does not exist`); - } - - watcher.unwatch(watchMapping.folderPath); - - watchMappings = watchMappings.filter( - (mapping) => mapping.folderPath !== watchMapping.folderPath, - ); - - setWatchMappings(watchMappings); }; -export function updateWatchMappingSyncedFiles( +export const watchUpdateIgnoredFiles = ( + ignoredFiles: FolderWatch["ignoredFiles"], folderPath: string, - files: FolderWatch["syncedFiles"], -): void { - const watchMappings = getWatchMappings(); - const watchMapping = watchMappings.find( - (mapping) => mapping.folderPath === folderPath, +) => { + setFolderWatches( + folderWatches().map((watch) => { + if (watch.folderPath == folderPath) { + watch.ignoredFiles = ignoredFiles; + } + return watch; + }), ); +}; - if (!watchMapping) { - throw Error(`Watch mapping not found`); +export const watchFindFiles = async (dirPath: string) => { + const items = await fs.readdir(dirPath, { withFileTypes: true }); + let paths: string[] = []; + for (const item of items) { + const itemPath = path.posix.join(dirPath, item.name); + if (item.isFile()) { + paths.push(itemPath); + } else if (item.isDirectory()) { + paths = [...paths, ...(await watchFindFiles(itemPath))]; + } } - - watchMapping.syncedFiles = files; - setWatchMappings(watchMappings); -} - -export function updateWatchMappingIgnoredFiles( - folderPath: string, - files: FolderWatch["ignoredFiles"], -): void { - const watchMappings = getWatchMappings(); - const watchMapping = watchMappings.find( - (mapping) => mapping.folderPath === folderPath, - ); - - if (!watchMapping) { - throw Error(`Watch mapping not found`); - } - - watchMapping.ignoredFiles = files; - setWatchMappings(watchMappings); -} - -export function getWatchMappings() { - const mappings = watchStore.get("mappings") ?? []; - return mappings; -} - -function setWatchMappings(watchMappings: WatchStoreType["mappings"]) { - watchStore.set("mappings", watchMappings); -} + return paths; +}; diff --git a/desktop/src/main/stores/keys.store.ts b/desktop/src/main/stores/keys.store.ts deleted file mode 100644 index 4f8618cea..000000000 --- a/desktop/src/main/stores/keys.store.ts +++ /dev/null @@ -1,18 +0,0 @@ -import Store, { Schema } from "electron-store"; -import type { KeysStoreType } from "../../types/main"; - -const keysStoreSchema: Schema = { - AnonymizeUserID: { - type: "object", - properties: { - id: { - type: "string", - }, - }, - }, -}; - -export const keysStore = new Store({ - name: "keys", - schema: keysStoreSchema, -}); diff --git a/desktop/src/main/stores/safeStorage.store.ts b/desktop/src/main/stores/safe-storage.ts similarity index 63% rename from desktop/src/main/stores/safeStorage.store.ts rename to desktop/src/main/stores/safe-storage.ts index da95df3be..1e1369db8 100644 --- a/desktop/src/main/stores/safeStorage.store.ts +++ b/desktop/src/main/stores/safe-storage.ts @@ -1,7 +1,10 @@ import Store, { Schema } from "electron-store"; -import type { SafeStorageStoreType } from "../../types/main"; -const safeStorageSchema: Schema = { +interface SafeStorageStore { + encryptionKey: string; +} + +const safeStorageSchema: Schema = { encryptionKey: { type: "string", }, diff --git a/desktop/src/main/stores/upload.store.ts b/desktop/src/main/stores/upload-status.ts similarity index 65% rename from desktop/src/main/stores/upload.store.ts rename to desktop/src/main/stores/upload-status.ts index 20b1f419d..25af7a49e 100644 --- a/desktop/src/main/stores/upload.store.ts +++ b/desktop/src/main/stores/upload-status.ts @@ -1,7 +1,12 @@ import Store, { Schema } from "electron-store"; -import type { UploadStoreType } from "../../types/main"; -const uploadStoreSchema: Schema = { +export interface UploadStatusStore { + filePaths: string[]; + zipPaths: string[]; + collectionName: string; +} + +const uploadStatusSchema: Schema = { filePaths: { type: "array", items: { @@ -21,5 +26,5 @@ const uploadStoreSchema: Schema = { export const uploadStatusStore = new Store({ name: "upload-status", - schema: uploadStoreSchema, + schema: uploadStatusSchema, }); diff --git a/desktop/src/main/stores/user-preferences.ts b/desktop/src/main/stores/user-preferences.ts index a305f1a99..b4a02bc5b 100644 --- a/desktop/src/main/stores/user-preferences.ts +++ b/desktop/src/main/stores/user-preferences.ts @@ -1,12 +1,12 @@ import Store, { Schema } from "electron-store"; -interface UserPreferencesSchema { +interface UserPreferences { hideDockIcon: boolean; skipAppVersion?: string; muteUpdateNotificationVersion?: string; } -const userPreferencesSchema: Schema = { +const userPreferencesSchema: Schema = { hideDockIcon: { type: "boolean", }, diff --git a/desktop/src/main/stores/watch.store.ts b/desktop/src/main/stores/watch.store.ts deleted file mode 100644 index 55470ce86..000000000 --- a/desktop/src/main/stores/watch.store.ts +++ /dev/null @@ -1,47 +0,0 @@ -import Store, { Schema } from "electron-store"; -import { WatchStoreType } from "../../types/ipc"; - -const watchStoreSchema: Schema = { - mappings: { - type: "array", - items: { - type: "object", - properties: { - rootFolderName: { - type: "string", - }, - uploadStrategy: { - type: "number", - }, - folderPath: { - type: "string", - }, - syncedFiles: { - type: "array", - items: { - type: "object", - properties: { - path: { - type: "string", - }, - id: { - type: "number", - }, - }, - }, - }, - ignoredFiles: { - type: "array", - items: { - type: "string", - }, - }, - }, - }, - }, -}; - -export const watchStore = new Store({ - name: "watch-status", - schema: watchStoreSchema, -}); diff --git a/desktop/src/main/stores/watch.ts b/desktop/src/main/stores/watch.ts new file mode 100644 index 000000000..7ee383038 --- /dev/null +++ b/desktop/src/main/stores/watch.ts @@ -0,0 +1,73 @@ +import Store, { Schema } from "electron-store"; +import { type FolderWatch } from "../../types/ipc"; +import log from "../log"; + +interface WatchStore { + mappings: FolderWatchWithLegacyFields[]; +} + +type FolderWatchWithLegacyFields = FolderWatch & { + /** @deprecated Only retained for migration, do not use in other code */ + rootFolderName?: string; + /** @deprecated Only retained for migration, do not use in other code */ + uploadStrategy?: number; +}; + +const watchStoreSchema: Schema = { + mappings: { + type: "array", + items: { + type: "object", + properties: { + rootFolderName: { type: "string" }, + collectionMapping: { type: "string" }, + uploadStrategy: { type: "number" }, + folderPath: { type: "string" }, + syncedFiles: { + type: "array", + items: { + type: "object", + properties: { + path: { type: "string" }, + uploadedFileID: { type: "number" }, + collectionID: { type: "number" }, + }, + }, + }, + ignoredFiles: { + type: "array", + items: { type: "string" }, + }, + }, + }, + }, +}; + +export const watchStore = new Store({ + name: "watch-status", + schema: watchStoreSchema, +}); + +/** + * Previous versions of the store used to store an integer to indicate the + * collection mapping, migrate these to the new schema if we encounter them. + */ +export const migrateLegacyWatchStoreIfNeeded = () => { + let needsUpdate = false; + const watches = watchStore.get("mappings")?.map((watch) => { + let collectionMapping = watch.collectionMapping; + if (!collectionMapping) { + collectionMapping = watch.uploadStrategy == 1 ? "parent" : "root"; + needsUpdate = true; + } + if (watch.rootFolderName) { + delete watch.rootFolderName; + needsUpdate = true; + } + return { ...watch, collectionMapping }; + }); + if (needsUpdate) { + watchStore.set("mappings", watches); + log.info("Migrated legacy watch store data to new schema"); + } +}; diff --git a/desktop/src/main/util.ts b/desktop/src/main/util.ts index d0c6699e9..b997d738e 100644 --- a/desktop/src/main/util.ts +++ b/desktop/src/main/util.ts @@ -56,6 +56,13 @@ export const openDirectory = async (dirPath: string) => { if (res) throw new Error(`Failed to open directory ${dirPath}: res`); }; +/** + * Open the app's log directory in the system's folder viewer. + * + * @see {@link openDirectory} + */ +export const openLogDirectory = () => openDirectory(logDirectoryPath()); + /** * Return the path where the logs for the app are saved. * @@ -72,10 +79,3 @@ export const openDirectory = async (dirPath: string) => { * */ const logDirectoryPath = () => app.getPath("logs"); - -/** - * Open the app's log directory in the system's folder viewer. - * - * @see {@link openDirectory} - */ -export const openLogDirectory = () => openDirectory(logDirectoryPath()); diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 2749fa50d..7d0df41d5 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -40,12 +40,13 @@ import { contextBridge, ipcRenderer } from "electron/renderer"; // While we can't import other code, we can import types since they're just -// needed when compiling and will not be needed / looked around for at runtime. +// needed when compiling and will not be needed or looked around for at runtime. import type { - AppUpdateInfo, + AppUpdate, + CollectionMapping, ElectronFile, - FILE_PATH_TYPE, FolderWatch, + PendingUploads, } from "./types/ipc"; // - General @@ -77,12 +78,12 @@ const onMainWindowFocus = (cb?: () => void) => { // - App update const onAppUpdateAvailable = ( - cb?: ((updateInfo: AppUpdateInfo) => void) | undefined, + cb?: ((update: AppUpdate) => void) | undefined, ) => { ipcRenderer.removeAllListeners("appUpdateAvailable"); if (cb) { - ipcRenderer.on("appUpdateAvailable", (_, updateInfo: AppUpdateInfo) => - cb(updateInfo), + ipcRenderer.on("appUpdateAvailable", (_, update: AppUpdate) => + cb(update), ); } }; @@ -118,6 +119,9 @@ const fsReadTextFile = (path: string): Promise => const fsWriteFile = (path: string, contents: string): Promise => ipcRenderer.invoke("fsWriteFile", path, contents); +const fsIsDir = (dirPath: string): Promise => + ipcRenderer.invoke("fsIsDir", dirPath); + // - AUDIT below this // - Conversion @@ -188,82 +192,78 @@ const showUploadZipDialog = (): Promise<{ // - Watch -const registerWatcherFunctions = ( - addFile: (file: ElectronFile) => Promise, - removeFile: (path: string) => Promise, - removeFolder: (folderPath: string) => Promise, -) => { - ipcRenderer.removeAllListeners("watch-add"); - ipcRenderer.removeAllListeners("watch-unlink"); - ipcRenderer.removeAllListeners("watch-unlink-dir"); - ipcRenderer.on("watch-add", (_, file: ElectronFile) => addFile(file)); - ipcRenderer.on("watch-unlink", (_, filePath: string) => - removeFile(filePath), - ); - ipcRenderer.on("watch-unlink-dir", (_, folderPath: string) => - removeFolder(folderPath), +const watchGet = (): Promise => ipcRenderer.invoke("watchGet"); + +const watchAdd = ( + folderPath: string, + collectionMapping: CollectionMapping, +): Promise => + ipcRenderer.invoke("watchAdd", folderPath, collectionMapping); + +const watchRemove = (folderPath: string): Promise => + ipcRenderer.invoke("watchRemove", folderPath); + +const watchUpdateSyncedFiles = ( + syncedFiles: FolderWatch["syncedFiles"], + folderPath: string, +): Promise => + ipcRenderer.invoke("watchUpdateSyncedFiles", syncedFiles, folderPath); + +const watchUpdateIgnoredFiles = ( + ignoredFiles: FolderWatch["ignoredFiles"], + folderPath: string, +): Promise => + ipcRenderer.invoke("watchUpdateIgnoredFiles", ignoredFiles, folderPath); + +const watchOnAddFile = (f: (path: string, watch: FolderWatch) => void) => { + ipcRenderer.removeAllListeners("watchAddFile"); + ipcRenderer.on("watchAddFile", (_, path: string, watch: FolderWatch) => + f(path, watch), ); }; -const addWatchMapping = ( - collectionName: string, - folderPath: string, - uploadStrategy: number, -): Promise => - ipcRenderer.invoke( - "addWatchMapping", - collectionName, - folderPath, - uploadStrategy, +const watchOnRemoveFile = (f: (path: string, watch: FolderWatch) => void) => { + ipcRenderer.removeAllListeners("watchRemoveFile"); + ipcRenderer.on("watchRemoveFile", (_, path: string, watch: FolderWatch) => + f(path, watch), ); +}; -const removeWatchMapping = (folderPath: string): Promise => - ipcRenderer.invoke("removeWatchMapping", folderPath); +const watchOnRemoveDir = (f: (path: string, watch: FolderWatch) => void) => { + ipcRenderer.removeAllListeners("watchRemoveDir"); + ipcRenderer.on("watchRemoveDir", (_, path: string, watch: FolderWatch) => + f(path, watch), + ); +}; -const getWatchMappings = (): Promise => - ipcRenderer.invoke("getWatchMappings"); - -const updateWatchMappingSyncedFiles = ( - folderPath: string, - files: FolderWatch["syncedFiles"], -): Promise => - ipcRenderer.invoke("updateWatchMappingSyncedFiles", folderPath, files); - -const updateWatchMappingIgnoredFiles = ( - folderPath: string, - files: FolderWatch["ignoredFiles"], -): Promise => - ipcRenderer.invoke("updateWatchMappingIgnoredFiles", folderPath, files); - -// - FS Legacy - -const isFolder = (dirPath: string): Promise => - ipcRenderer.invoke("isFolder", dirPath); +const watchFindFiles = (folderPath: string): Promise => + ipcRenderer.invoke("watchFindFiles", folderPath); // - Upload -const getPendingUploads = (): Promise<{ - files: ElectronFile[]; - collectionName: string; - type: string; -}> => ipcRenderer.invoke("getPendingUploads"); +const pendingUploads = (): Promise => + ipcRenderer.invoke("pendingUploads"); -const setToUploadFiles = ( - type: FILE_PATH_TYPE, +const setPendingUploadCollection = (collectionName: string): Promise => + ipcRenderer.invoke("setPendingUploadCollection", collectionName); + +const setPendingUploadFiles = ( + type: PendingUploads["type"], filePaths: string[], -): Promise => ipcRenderer.invoke("setToUploadFiles", type, filePaths); +): Promise => + ipcRenderer.invoke("setPendingUploadFiles", type, filePaths); + +// - const getElectronFilesFromGoogleZip = ( filePath: string, ): Promise => ipcRenderer.invoke("getElectronFilesFromGoogleZip", filePath); -const setToUploadCollection = (collectionName: string): Promise => - ipcRenderer.invoke("setToUploadCollection", collectionName); - const getDirFiles = (dirPath: string): Promise => ipcRenderer.invoke("getDirFiles", dirPath); +// // These objects exposed here will become available to the JS code in our // renderer (the web/ code) as `window.ElectronAPIs.*` // @@ -295,10 +295,13 @@ const getDirFiles = (dirPath: string): Promise => // https://www.electronjs.org/docs/latest/api/context-bridge#methods // // The copy itself is relatively fast, but the problem with transfering large -// amounts of data is potentially running out of memory during the copy. For an -// alternative, see [Note: IPC streams]. +// amounts of data is potentially running out of memory during the copy. +// +// For an alternative, see [Note: IPC streams]. +// contextBridge.exposeInMainWorld("electron", { // - General + appVersion, logToDisk, openDirectory, @@ -309,12 +312,14 @@ contextBridge.exposeInMainWorld("electron", { onMainWindowFocus, // - App update + onAppUpdateAvailable, updateAndRestart, updateOnNextRestart, skipAppUpdate, // - FS + fs: { exists: fsExists, rename: fsRename, @@ -323,42 +328,51 @@ contextBridge.exposeInMainWorld("electron", { rm: fsRm, readTextFile: fsReadTextFile, writeFile: fsWriteFile, + isDir: fsIsDir, }, // - Conversion + convertToJPEG, generateImageThumbnail, runFFmpegCmd, // - ML + clipImageEmbedding, clipTextEmbedding, detectFaces, faceEmbedding, // - File selection + selectDirectory, showUploadFilesDialog, showUploadDirsDialog, showUploadZipDialog, // - Watch - registerWatcherFunctions, - addWatchMapping, - removeWatchMapping, - getWatchMappings, - updateWatchMappingSyncedFiles, - updateWatchMappingIgnoredFiles, - // - FS legacy - // TODO: Move these into fs + document + rename if needed - isFolder, + watch: { + get: watchGet, + add: watchAdd, + remove: watchRemove, + onAddFile: watchOnAddFile, + onRemoveFile: watchOnRemoveFile, + onRemoveDir: watchOnRemoveDir, + findFiles: watchFindFiles, + updateSyncedFiles: watchUpdateSyncedFiles, + updateIgnoredFiles: watchUpdateIgnoredFiles, + }, // - Upload - getPendingUploads, - setToUploadFiles, + pendingUploads, + setPendingUploadCollection, + setPendingUploadFiles, + + // - + getElectronFilesFromGoogleZip, - setToUploadCollection, getDirFiles, }); diff --git a/desktop/src/types/ipc.ts b/desktop/src/types/ipc.ts index 3dae605a8..d96341982 100644 --- a/desktop/src/types/ipc.ts +++ b/desktop/src/types/ipc.ts @@ -5,20 +5,32 @@ * See [Note: types.ts <-> preload.ts <-> ipc.ts] */ +export interface AppUpdate { + autoUpdatable: boolean; + version: string; +} + export interface FolderWatch { - rootFolderName: string; - uploadStrategy: number; + collectionMapping: CollectionMapping; folderPath: string; syncedFiles: FolderWatchSyncedFile[]; ignoredFiles: string[]; } +export type CollectionMapping = "root" | "parent"; + export interface FolderWatchSyncedFile { path: string; uploadedFileID: number; collectionID: number; } +export interface PendingUploads { + collectionName: string; + type: "files" | "zips"; + files: ElectronFile[]; +} + /** * Errors that have special semantics on the web side. * @@ -65,18 +77,3 @@ export interface ElectronFile { blob: () => Promise; arrayBuffer: () => Promise; } - -export interface WatchStoreType { - mappings: FolderWatch[]; -} - -export enum FILE_PATH_TYPE { - /* eslint-disable no-unused-vars */ - FILES = "files", - ZIPS = "zips", -} - -export interface AppUpdateInfo { - autoUpdatable: boolean; - version: string; -} diff --git a/desktop/src/types/main.ts b/desktop/src/types/main.ts deleted file mode 100644 index 546749c54..000000000 --- a/desktop/src/types/main.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { FILE_PATH_TYPE } from "./ipc"; - -export interface AutoLauncherClient { - isEnabled: () => Promise; - toggleAutoLaunch: () => Promise; - wasAutoLaunched: () => Promise; -} - -export interface UploadStoreType { - filePaths: string[]; - zipPaths: string[]; - collectionName: string; -} - -export interface KeysStoreType { - AnonymizeUserID: { - id: string; - }; -} - -/* eslint-disable no-unused-vars */ -export const FILE_PATH_KEYS: { - [k in FILE_PATH_TYPE]: keyof UploadStoreType; -} = { - [FILE_PATH_TYPE.ZIPS]: "zipPaths", - [FILE_PATH_TYPE.FILES]: "filePaths", -}; - -export interface SafeStorageStoreType { - encryptionKey: string; -} diff --git a/web/apps/cast/src/types/upload/index.ts b/web/apps/cast/src/types/upload/index.ts index ef44b4a23..0e249846a 100644 --- a/web/apps/cast/src/types/upload/index.ts +++ b/web/apps/cast/src/types/upload/index.ts @@ -95,13 +95,6 @@ export interface ParsedExtractedMetadata { height: number; } -// This is used to prompt the user the make upload strategy choice -export interface ImportSuggestion { - rootFolderName: string; - hasNestedFolders: boolean; - hasRootLevelFileWithFolder: boolean; -} - export interface PublicUploadProps { token: string; passwordToken: string; diff --git a/web/apps/photos/src/components/Sidebar/UtilitySection.tsx b/web/apps/photos/src/components/Sidebar/UtilitySection.tsx index c9c734cd9..6b4a6f43d 100644 --- a/web/apps/photos/src/components/Sidebar/UtilitySection.tsx +++ b/web/apps/photos/src/components/Sidebar/UtilitySection.tsx @@ -206,7 +206,12 @@ export default function UtilitySection({ closeSidebar }) { closeSidebar={closeSidebar} setLoading={startLoading} /> - + {isElectron() && ( + + )} void; +interface CollectionMappingChoiceModalProps { open: boolean; onClose: () => void; - uploadToSingleCollection: () => void; + didSelect: (mapping: CollectionMapping) => void; } -function UploadStrategyChoiceModal({ - uploadToMultipleCollection, - uploadToSingleCollection, - ...props -}: Props) { - const handleClose = dialogCloseHandler({ - onClose: props.onClose, - }); + +export const CollectionMappingChoiceModal: React.FC< + CollectionMappingChoiceModalProps +> = ({ open, onClose, didSelect }) => { + const handleClose = dialogCloseHandler({ onClose }); return ( - + {t("MULTI_FOLDER_UPLOAD")} @@ -39,8 +36,8 @@ function UploadStrategyChoiceModal({ size="medium" color="accent" onClick={() => { - props.onClose(); - uploadToSingleCollection(); + onClose(); + didSelect("root"); }} > {t("UPLOAD_STRATEGY_SINGLE_COLLECTION")} @@ -52,8 +49,8 @@ function UploadStrategyChoiceModal({ size="medium" color="accent" onClick={() => { - props.onClose(); - uploadToMultipleCollection(); + onClose(); + didSelect("parent"); }} > {t("UPLOAD_STRATEGY_COLLECTION_PER_FOLDER")} @@ -62,5 +59,4 @@ function UploadStrategyChoiceModal({ ); -} -export default UploadStrategyChoiceModal; +}; diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index bb3d4fd9d..a622953b3 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -1,15 +1,11 @@ +import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; -import type { Electron } from "@/next/types/ipc"; +import type { CollectionMapping, Electron } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; import { isPromise } from "@ente/shared/utils"; import DiscFullIcon from "@mui/icons-material/DiscFull"; import UserNameInputDialog from "components/UserNameInputDialog"; -import { - DEFAULT_IMPORT_SUGGESTION, - PICKED_UPLOAD_TYPE, - UPLOAD_STAGES, - UPLOAD_STRATEGY, -} from "constants/upload"; +import { PICKED_UPLOAD_TYPE, UPLOAD_STAGES } from "constants/upload"; import { t } from "i18next"; import isElectron from "is-electron"; import { AppContext } from "pages/_app"; @@ -17,14 +13,14 @@ import { GalleryContext } from "pages/gallery"; import { useContext, useEffect, useRef, useState } from "react"; import billingService from "services/billingService"; import { getLatestCollections } from "services/collectionService"; -import ImportService from "services/importService"; +import { setToUploadCollection } from "services/pending-uploads"; import { getPublicCollectionUID, getPublicCollectionUploaderName, savePublicCollectionUploaderName, } from "services/publicCollectionService"; import uploadManager from "services/upload/uploadManager"; -import watchFolderService from "services/watch"; +import watcher from "services/watch"; import { NotificationAttributes } from "types/Notification"; import { Collection } from "types/collection"; import { @@ -35,11 +31,7 @@ import { SetLoading, UploadTypeSelectorIntent, } from "types/gallery"; -import { - ElectronFile, - FileWithCollection, - ImportSuggestion, -} from "types/upload"; +import { ElectronFile, FileWithCollection } from "types/upload"; import { InProgressUpload, SegregatedFinishedUploads, @@ -53,13 +45,15 @@ import { getRootLevelFileWithFolderNotAllowMessage, } from "utils/ui"; import { + DEFAULT_IMPORT_SUGGESTION, filterOutSystemFiles, getImportSuggestion, groupFilesBasedOnParentFolder, + type ImportSuggestion, } from "utils/upload"; import { SetCollectionNamerAttributes } from "../Collections/CollectionNamer"; +import { CollectionMappingChoiceModal } from "./CollectionMappingChoiceModal"; import UploadProgress from "./UploadProgress"; -import UploadStrategyChoiceModal from "./UploadStrategyChoiceModal"; import UploadTypeSelector from "./UploadTypeSelector"; const FIRST_ALBUM_NAME = "My First Album"; @@ -137,6 +131,7 @@ export default function Uploader(props: Props) { const closeUploadProgress = () => setUploadProgressView(false); const showUserNameInputDialog = () => setUserNameInputDialogView(true); + // eslint-disable-next-line @typescript-eslint/no-unused-vars const setCollectionName = (collectionName: string) => { isPendingDesktopUpload.current = true; pendingDesktopUploadCollectionName.current = collectionName; @@ -177,18 +172,27 @@ export default function Uploader(props: Props) { } if (isElectron()) { - ImportService.getPendingUploads().then( - ({ files: electronFiles, collectionName, type }) => { - log.info(`found pending desktop upload, resuming uploads`); - resumeDesktopUpload(type, electronFiles, collectionName); - }, - ); - watchFolderService.init( + ensureElectron() + .pendingUploads() + .then((pending) => { + if (pending) { + log.info("Resuming pending desktop upload", pending); + resumeDesktopUpload( + pending.type == "files" + ? PICKED_UPLOAD_TYPE.FILES + : PICKED_UPLOAD_TYPE.ZIPS, + pending.files, + pending.collectionName, + ); + } + }); + /* TODO(MR): This is the connection point, implement + watcher.init( setElectronFiles, setCollectionName, props.syncWithRemote, - appContext.setIsFolderSyncRunning, ); + */ } }, [ publicCollectionGalleryContext.accessedThroughSharedURL, @@ -291,18 +295,16 @@ export default function Uploader(props: Props) { }`, ); if (uploadManager.isUploadRunning()) { - if (watchFolderService.isUploadRunning()) { + if (watcher.isUploadRunning()) { + // Pause watch folder sync on user upload log.info( - "watchFolder upload was running, pausing it to run user upload", + "Folder watcher was uploading, pausing it to first run user upload", ); - // pause watch folder service on user upload - watchFolderService.pauseRunningSync(); + watcher.pauseRunningSync(); } else { log.info( - "an upload is already running, rejecting new upload request", + "Ignoring new upload request because an upload is already running", ); - // no-op - // a user upload is already in progress return; } } @@ -330,7 +332,7 @@ export default function Uploader(props: Props) { const importSuggestion = getImportSuggestion( pickedUploadType.current, - toUploadFiles.current, + toUploadFiles.current.map((file) => file["path"]), ); setImportSuggestion(importSuggestion); @@ -391,7 +393,7 @@ export default function Uploader(props: Props) { }; const uploadFilesToNewCollections = async ( - strategy: UPLOAD_STRATEGY, + strategy: CollectionMapping, collectionName?: string, ) => { try { @@ -405,7 +407,7 @@ export default function Uploader(props: Props) { string, (File | ElectronFile)[] >(); - if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) { + if (strategy == "root") { collectionNameToFilesMap.set( collectionName, toUploadFiles.current, @@ -505,18 +507,19 @@ export default function Uploader(props: Props) { if ( electron && !isPendingDesktopUpload.current && - !watchFolderService.isUploadRunning() + !watcher.isUploadRunning() ) { - await ImportService.setToUploadCollection(collections); + await setToUploadCollection(collections); + // TODO (MR): What happens when we have both? if (zipPaths.current) { - await electron.setToUploadFiles( - PICKED_UPLOAD_TYPE.ZIPS, + await electron.setPendingUploadFiles( + "zips", zipPaths.current, ); zipPaths.current = null; } - await electron.setToUploadFiles( - PICKED_UPLOAD_TYPE.FILES, + await electron.setPendingUploadFiles( + "files", filesWithCollectionToUploadIn.map( ({ file }) => (file as ElectronFile).path, ), @@ -532,14 +535,14 @@ export default function Uploader(props: Props) { closeUploadProgress(); } if (isElectron()) { - if (watchFolderService.isUploadRunning()) { - await watchFolderService.allFileUploadsDone( + if (watcher.isUploadRunning()) { + await watcher.allFileUploadsDone( filesWithCollectionToUploadIn, collections, ); - } else if (watchFolderService.isSyncPaused()) { + } else if (watcher.isSyncPaused()) { // resume the service after user upload is done - watchFolderService.resumePausedSync(); + watcher.resumePausedSync(); } } } catch (e) { @@ -605,10 +608,7 @@ export default function Uploader(props: Props) { } const uploadToSingleNewCollection = (collectionName: string) => { - uploadFilesToNewCollections( - UPLOAD_STRATEGY.SINGLE_COLLECTION, - collectionName, - ); + uploadFilesToNewCollections("root", collectionName); }; const showCollectionCreateModal = (suggestedName: string) => { @@ -647,7 +647,7 @@ export default function Uploader(props: Props) { `upload pending files to collection - ${pendingDesktopUploadCollectionName.current}`, ); uploadFilesToNewCollections( - UPLOAD_STRATEGY.SINGLE_COLLECTION, + "root", pendingDesktopUploadCollectionName.current, ); pendingDesktopUploadCollectionName.current = null; @@ -655,17 +655,13 @@ export default function Uploader(props: Props) { log.info( `pending upload - strategy - "multiple collections" `, ); - uploadFilesToNewCollections( - UPLOAD_STRATEGY.COLLECTION_PER_FOLDER, - ); + uploadFilesToNewCollections("parent"); } return; } if (isElectron() && pickedUploadType === PICKED_UPLOAD_TYPE.ZIPS) { log.info("uploading zip files"); - uploadFilesToNewCollections( - UPLOAD_STRATEGY.COLLECTION_PER_FOLDER, - ); + uploadFilesToNewCollections("parent"); return; } if (isFirstUpload && !importSuggestion.rootFolderName) { @@ -784,16 +780,26 @@ export default function Uploader(props: Props) { ); return; } - uploadFilesToNewCollections(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER); + uploadFilesToNewCollections("parent"); + }; + + const didSelectCollectionMapping = (mapping: CollectionMapping) => { + switch (mapping) { + case "root": + handleUploadToSingleCollection(); + break; + case "parent": + handleUploadToMultipleCollections(); + break; + } }; return ( <> - void; } +/** + * View the state of and manage folder watches. + * + * This is the screen that controls that "watch folder" feature in the app. + */ export const WatchFolder: React.FC = ({ open, onClose }) => { - const [mappings, setMappings] = useState([]); - const [inputFolderPath, setInputFolderPath] = useState(""); + // The folders we are watching + const [watches, setWatches] = useState(); + // Temporarily stash the folder path while we show a choice dialog to the + // user to select the collection mapping. + const [savedFolderPath, setSavedFolderPath] = useState< + string | undefined + >(); + // True when we're showing the choice dialog to ask the user to set the + // collection mapping. const [choiceModalOpen, setChoiceModalOpen] = useState(false); + const appContext = useContext(AppContext); - const electron = globalThis.electron; - useEffect(() => { - if (!electron) return; - watchFolderService.getWatchMappings().then((m) => setMappings(m)); + watcher.getWatches().then((ws) => setWatches(ws)); }, []); useEffect(() => { @@ -64,69 +76,41 @@ export const WatchFolder: React.FC = ({ open, onClose }) => { for (let i = 0; i < folders.length; i++) { const folder: any = folders[i]; const path = (folder.path as string).replace(/\\/g, "/"); - if (await watchFolderService.isFolder(path)) { - await addFolderForWatching(path); + if (await ensureElectron().fs.isDir(path)) { + await selectCollectionMappingAndAddWatch(path); } } }; - const addFolderForWatching = async (path: string) => { - if (!electron) return; - - setInputFolderPath(path); - const files = await electron.getDirFiles(path); - const analysisResult = getImportSuggestion( - PICKED_UPLOAD_TYPE.FOLDERS, - files, - ); - if (analysisResult.hasNestedFolders) { - setChoiceModalOpen(true); + const selectCollectionMappingAndAddWatch = async (path: string) => { + const filePaths = await ensureElectron().watch.findFiles(path); + if (areAllInSameDirectory(filePaths)) { + addWatch(path, "root"); } else { - handleAddWatchMapping(UPLOAD_STRATEGY.SINGLE_COLLECTION, path); + setSavedFolderPath(path); + setChoiceModalOpen(true); } }; - const handleAddFolderClick = async () => { - await handleFolderSelection(); - }; + const addWatch = (folderPath: string, mapping: CollectionMapping) => + watcher.addWatch(folderPath, mapping).then((ws) => setWatches(ws)); - const handleFolderSelection = async () => { - const folderPath = await watchFolderService.selectFolder(); - if (folderPath) { - await addFolderForWatching(folderPath); + const addNewWatch = async () => { + const dirPath = await ensureElectron().selectDirectory(); + if (dirPath) { + await selectCollectionMappingAndAddWatch(dirPath); } }; - const handleAddWatchMapping = async ( - uploadStrategy: UPLOAD_STRATEGY, - folderPath?: string, - ) => { - folderPath = folderPath || inputFolderPath; - await watchFolderService.addWatchMapping( - folderPath.substring(folderPath.lastIndexOf("/") + 1), - folderPath, - uploadStrategy, - ); - setInputFolderPath(""); - setMappings(await watchFolderService.getWatchMappings()); - }; - - const handleRemoveWatchMapping = (mapping: WatchMapping) => { - watchFolderService - .mappingsAfterRemovingFolder(mapping.folderPath) - .then((ms) => setMappings(ms)); - }; + const removeWatch = async (watch: FolderWatch) => + watcher.removeWatch(watch.folderPath).then((ws) => setWatches(ws)); const closeChoiceModal = () => setChoiceModalOpen(false); - const uploadToSingleCollection = () => { + const addWatchWithMapping = (mapping: CollectionMapping) => { closeChoiceModal(); - handleAddWatchMapping(UPLOAD_STRATEGY.SINGLE_COLLECTION); - }; - - const uploadToMultipleCollection = () => { - closeChoiceModal(); - handleAddWatchMapping(UPLOAD_STRATEGY.COLLECTION_PER_FOLDER); + setSavedFolderPath(undefined); + addWatch(ensure(savedFolderPath), mapping); }; return ( @@ -144,15 +128,8 @@ export const WatchFolder: React.FC = ({ open, onClose }) => { - - - ); }; -const MappingsContainer = styled(Box)(() => ({ +interface WatchList { + watches: FolderWatch[]; + removeWatch: (watch: FolderWatch) => void; +} + +const WatchList: React.FC = ({ watches, removeWatch }) => { + return watches.length === 0 ? ( + + ) : ( + + {watches.map((watch) => { + return ( + + ); + })} + + ); +}; + +const WatchesContainer = styled(Box)(() => ({ height: "278px", overflow: "auto", "&::-webkit-scrollbar": { @@ -182,47 +181,9 @@ const MappingsContainer = styled(Box)(() => ({ }, })); -const NoMappingsContainer = styled(VerticallyCentered)({ - textAlign: "left", - alignItems: "flex-start", - marginBottom: "32px", -}); - -const EntryContainer = styled(Box)({ - marginLeft: "12px", - marginRight: "6px", - marginBottom: "12px", -}); - -interface MappingListProps { - mappings: WatchMapping[]; - handleRemoveWatchMapping: (value: WatchMapping) => void; -} - -const MappingList: React.FC = ({ - mappings, - handleRemoveWatchMapping, -}) => { - return mappings.length === 0 ? ( - - ) : ( - - {mappings.map((mapping) => { - return ( - - ); - })} - - ); -}; - -const NoMappingsContent: React.FC = () => { +const NoWatches: React.FC = () => { return ( - + {t("NO_FOLDERS_ADDED")} @@ -243,10 +204,16 @@ const NoMappingsContent: React.FC = () => { - + ); }; +const NoWatchesContainer = styled(VerticallyCentered)({ + textAlign: "left", + alignItems: "flex-start", + marginBottom: "32px", +}); + const CheckmarkIcon: React.FC = () => { return ( { sx={{ display: "inline", fontSize: "15px", - color: (theme) => theme.palette.secondary.main, }} /> ); }; -interface MappingEntryProps { - mapping: WatchMapping; - handleRemoveMapping: (mapping: WatchMapping) => void; +interface WatchEntryProps { + watch: FolderWatch; + removeWatch: (watch: FolderWatch) => void; } -const MappingEntry: React.FC = ({ - mapping, - handleRemoveMapping, -}) => { +const WatchEntry: React.FC = ({ watch, removeWatch }) => { const appContext = React.useContext(AppContext); - const stopWatching = () => { - handleRemoveMapping(mapping); - }; - const confirmStopWatching = () => { appContext.setDialogMessage({ title: t("STOP_WATCHING_FOLDER"), @@ -285,7 +244,7 @@ const MappingEntry: React.FC = ({ variant: "secondary", }, proceed: { - action: stopWatching, + action: () => removeWatch(watch), text: t("YES_STOP"), variant: "critical", }, @@ -295,8 +254,7 @@ const MappingEntry: React.FC = ({ return ( - {mapping && - mapping.uploadStrategy === UPLOAD_STRATEGY.SINGLE_COLLECTION ? ( + {watch.collectionMapping === "root" ? ( @@ -306,41 +264,45 @@ const MappingEntry: React.FC = ({ )} - + - {mapping.folderPath} + {watch.folderPath} - + ); }; +const EntryContainer = styled(Box)({ + marginLeft: "12px", + marginRight: "6px", + marginBottom: "12px", +}); + interface EntryHeadingProps { - mapping: WatchMapping; + watch: FolderWatch; } -const EntryHeading: React.FC = ({ mapping }) => { - const appContext = useContext(AppContext); +const EntryHeading: React.FC = ({ watch }) => { + const folderPath = watch.folderPath; + return ( - {mapping.rootFolderName} - {appContext.isFolderSyncRunning && - watchFolderService.isMappingSyncInProgress(mapping) && ( - - )} + {basename(folderPath)} + {watcher.isSyncingFolder(folderPath) && ( + + )} ); }; -interface MappingEntryOptionsProps { +interface EntryOptionsProps { confirmStopWatching: () => void; } -const MappingEntryOptions: React.FC = ({ - confirmStopWatching, -}) => { +const EntryOptions: React.FC = ({ confirmStopWatching }) => { return ( void; setDialogMessage: SetDialogBoxAttributes; setNotificationAttributes: SetNotificationAttributes; - isFolderSyncRunning: boolean; - setIsFolderSyncRunning: (isRunning: boolean) => void; watchFolderView: boolean; setWatchFolderView: (isOpen: boolean) => void; watchFolderFiles: FileList; @@ -128,7 +126,6 @@ export default function App({ Component, pageProps }: AppProps) { useState(null); const [messageDialogView, setMessageDialogView] = useState(false); const [dialogBoxV2View, setDialogBoxV2View] = useState(false); - const [isFolderSyncRunning, setIsFolderSyncRunning] = useState(false); const [watchFolderView, setWatchFolderView] = useState(false); const [watchFolderFiles, setWatchFolderFiles] = useState(null); const isMobile = useMediaQuery("(max-width:428px)"); @@ -160,9 +157,9 @@ export default function App({ Component, pageProps }: AppProps) { const electron = globalThis.electron; if (!electron) return; - const showUpdateDialog = (updateInfo: AppUpdateInfo) => { - if (updateInfo.autoUpdatable) { - setDialogMessage(getUpdateReadyToInstallMessage(updateInfo)); + const showUpdateDialog = (update: AppUpdate) => { + if (update.autoUpdatable) { + setDialogMessage(getUpdateReadyToInstallMessage(update)); } else { setNotificationAttributes({ endIcon: , @@ -170,7 +167,7 @@ export default function App({ Component, pageProps }: AppProps) { message: t("UPDATE_AVAILABLE"), onClick: () => setDialogMessage( - getUpdateAvailableForDownloadMessage(updateInfo), + getUpdateAvailableForDownloadMessage(update), ), }); } @@ -403,8 +400,6 @@ export default function App({ Component, pageProps }: AppProps) { finishLoading, closeMessageDialog, setDialogMessage, - isFolderSyncRunning, - setIsFolderSyncRunning, watchFolderView, setWatchFolderView, watchFolderFiles, diff --git a/web/apps/photos/src/services/importService.ts b/web/apps/photos/src/services/importService.ts deleted file mode 100644 index 6d2c46a85..000000000 --- a/web/apps/photos/src/services/importService.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { ensureElectron } from "@/next/electron"; -import log from "@/next/log"; -import { PICKED_UPLOAD_TYPE } from "constants/upload"; -import { Collection } from "types/collection"; -import { ElectronFile, FileWithCollection } from "types/upload"; - -interface PendingUploads { - files: ElectronFile[]; - collectionName: string; - type: PICKED_UPLOAD_TYPE; -} - -class ImportService { - async getPendingUploads(): Promise { - try { - const pendingUploads = - (await ensureElectron().getPendingUploads()) as PendingUploads; - return pendingUploads; - } catch (e) { - if (e?.message?.includes("ENOENT: no such file or directory")) { - // ignore - } else { - log.error("failed to getPendingUploads ", e); - } - return { files: [], collectionName: null, type: null }; - } - } - - async setToUploadCollection(collections: Collection[]) { - let collectionName: string = null; - /* collection being one suggest one of two things - 1. Either the user has upload to a single existing collection - 2. Created a new single collection to upload to - may have had multiple folder, but chose to upload - to one album - hence saving the collection name when upload collection count is 1 - helps the info of user choosing this options - and on next upload we can directly start uploading to this collection - */ - if (collections.length === 1) { - collectionName = collections[0].name; - } - await ensureElectron().setToUploadCollection(collectionName); - } - - async updatePendingUploads(files: FileWithCollection[]) { - const filePaths = []; - for (const fileWithCollection of files) { - if (fileWithCollection.isLivePhoto) { - filePaths.push( - (fileWithCollection.livePhotoAssets.image as ElectronFile) - .path, - (fileWithCollection.livePhotoAssets.video as ElectronFile) - .path, - ); - } else { - filePaths.push((fileWithCollection.file as ElectronFile).path); - } - } - await ensureElectron().setToUploadFiles( - PICKED_UPLOAD_TYPE.FILES, - filePaths, - ); - } - - async cancelRemainingUploads() { - const electron = ensureElectron(); - await electron.setToUploadCollection(null); - await electron.setToUploadFiles(PICKED_UPLOAD_TYPE.ZIPS, []); - await electron.setToUploadFiles(PICKED_UPLOAD_TYPE.FILES, []); - } -} - -export default new ImportService(); diff --git a/web/apps/photos/src/services/pending-uploads.ts b/web/apps/photos/src/services/pending-uploads.ts new file mode 100644 index 000000000..3b219f5b0 --- /dev/null +++ b/web/apps/photos/src/services/pending-uploads.ts @@ -0,0 +1,42 @@ +import { ensureElectron } from "@/next/electron"; +import { Collection } from "types/collection"; +import { ElectronFile, FileWithCollection } from "types/upload"; + +export const setToUploadCollection = async (collections: Collection[]) => { + let collectionName: string = null; + /* collection being one suggest one of two things + 1. Either the user has upload to a single existing collection + 2. Created a new single collection to upload to + may have had multiple folder, but chose to upload + to one album + hence saving the collection name when upload collection count is 1 + helps the info of user choosing this options + and on next upload we can directly start uploading to this collection + */ + if (collections.length === 1) { + collectionName = collections[0].name; + } + await ensureElectron().setPendingUploadCollection(collectionName); +}; + +export const updatePendingUploads = async (files: FileWithCollection[]) => { + const filePaths = []; + for (const fileWithCollection of files) { + if (fileWithCollection.isLivePhoto) { + filePaths.push( + (fileWithCollection.livePhotoAssets.image as ElectronFile).path, + (fileWithCollection.livePhotoAssets.video as ElectronFile).path, + ); + } else { + filePaths.push((fileWithCollection.file as ElectronFile).path); + } + } + await ensureElectron().setPendingUploadFiles("files", filePaths); +}; + +export const cancelRemainingUploads = async () => { + const electron = ensureElectron(); + await electron.setPendingUploadCollection(undefined); + await electron.setPendingUploadFiles("zips", []); + await electron.setPendingUploadFiles("files", []); +}; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index d222999d8..e927d7b08 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -8,13 +8,16 @@ import { Events, eventBus } from "@ente/shared/events"; import { Remote } from "comlink"; import { UPLOAD_RESULT, UPLOAD_STAGES } from "constants/upload"; import isElectron from "is-electron"; -import ImportService from "services/importService"; +import { + cancelRemainingUploads, + updatePendingUploads, +} from "services/pending-uploads"; import { getLocalPublicFiles, getPublicCollectionUID, } from "services/publicCollectionService"; import { getDisableCFUploadProxyFlag } from "services/userService"; -import watchFolderService from "services/watch"; +import watcher from "services/watch"; import { Collection } from "types/collection"; import { EncryptedEnteFile, EnteFile } from "types/file"; import { SetFiles } from "types/gallery"; @@ -177,7 +180,7 @@ class UploadManager { if (e.message === CustomError.UPLOAD_CANCELLED) { if (isElectron()) { this.remainingFiles = []; - await ImportService.cancelRemainingUploads(); + await cancelRemainingUploads(); } } else { log.error("uploading failed with error", e); @@ -387,11 +390,13 @@ class UploadManager { uploadedFile: EncryptedEnteFile, ) { if (isElectron()) { - await watchFolderService.onFileUpload( - fileUploadResult, - fileWithCollection, - uploadedFile, - ); + if (watcher.isUploadRunning()) { + await watcher.onFileUpload( + fileUploadResult, + fileWithCollection, + uploadedFile, + ); + } } } @@ -431,12 +436,12 @@ class UploadManager { this.remainingFiles = this.remainingFiles.filter( (file) => !areFileWithCollectionsSame(file, fileWithCollection), ); - await ImportService.updatePendingUploads(this.remainingFiles); + await updatePendingUploads(this.remainingFiles); } } public shouldAllowNewUpload = () => { - return !this.uploadInProgress || watchFolderService.isUploadRunning(); + return !this.uploadInProgress || watcher.isUploadRunning(); }; } diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index 2d5ef0228..703dd87ad 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -4,283 +4,318 @@ */ import { ensureElectron } from "@/next/electron"; +import { basename, dirname } from "@/next/file"; import log from "@/next/log"; -import { UPLOAD_RESULT, UPLOAD_STRATEGY } from "constants/upload"; +import type { + CollectionMapping, + FolderWatch, + FolderWatchSyncedFile, +} from "@/next/types/ipc"; +import { UPLOAD_RESULT } from "constants/upload"; import debounce from "debounce"; import uploadManager from "services/upload/uploadManager"; import { Collection } from "types/collection"; import { EncryptedEnteFile } from "types/file"; import { ElectronFile, FileWithCollection } from "types/upload"; -import { - EventQueueItem, - WatchMapping, - WatchMappingSyncedFile, -} from "types/watchFolder"; import { groupFilesBasedOnCollectionID } from "utils/file"; -import { isSystemFile } from "utils/upload"; +import { isHiddenFile } from "utils/upload"; import { removeFromCollection } from "./collectionService"; import { getLocalFiles } from "./fileService"; -class WatchFolderService { - private eventQueue: EventQueueItem[] = []; - private currentEvent: EventQueueItem; - private currentlySyncedMapping: WatchMapping; - private trashingDirQueue: string[] = []; - private isEventRunning: boolean = false; - private uploadRunning: boolean = false; +/** + * Watch for file system folders and automatically update the corresponding Ente + * collections. + * + * This class relies on APIs exposed over the Electron IPC layer, and thus only + * works when we're running inside our desktop app. + */ +class FolderWatcher { + /** Pending file system events that we need to process. */ + private eventQueue: WatchEvent[] = []; + /** The folder watch whose event we're currently processing */ + private activeWatch: FolderWatch | undefined; + /** + * If the file system directory corresponding to the (root) folder path of a + * folder watch is deleted on disk, we note down that in this queue so that + * we can ignore any file system events that come for it next. + * + * TODO: is this really needed? the mappings are pre-checked first. + */ + private deletedFolderPaths: string[] = []; + /** `true` if we are using the uploader. */ + private uploadRunning = false; + /** `true` if we are temporarily paused to let a user upload go through. */ + private isPaused = false; private filePathToUploadedFileIDMap = new Map(); private unUploadableFilePaths = new Set(); - private isPaused = false; - private setElectronFiles: (files: ElectronFile[]) => void; - private setCollectionName: (collectionName: string) => void; + + /** + * A function to call when we want to enqueue a new upload of the given list + * of file paths to the given Ente collection. + * + * This is passed as a param to {@link init}. + */ + private upload: (collectionName: string, filePaths: string[]) => void; + /** + * A function to call when we want to sync with the backend. + * + * This is passed as a param to {@link init}. + */ private syncWithRemote: () => void; - private setWatchFolderServiceIsRunning: (isRunning: boolean) => void; + + /** A helper function that debounces invocations of {@link runNextEvent}. */ private debouncedRunNextEvent: () => void; constructor() { this.debouncedRunNextEvent = debounce(() => this.runNextEvent(), 1000); } + /** + * Initialize the watcher and start processing file system events. + * + * This is only called when we're running in the context of our desktop app. + * + * The caller provides us with the hooks we can use to actually upload the + * files, and to sync with remote (say after deletion). + */ + init( + upload: (collectionName: string, filePaths: string[]) => void, + syncWithRemote: () => void, + ) { + this.upload = upload; + this.syncWithRemote = syncWithRemote; + this.registerListeners(); + this.syncWithDisk(); + } + + /** `true` if we are currently using the uploader */ isUploadRunning() { return this.uploadRunning; } + /** `true` if syncing has been temporarily paused */ isSyncPaused() { return this.isPaused; } - async init( - setElectronFiles: (files: ElectronFile[]) => void, - setCollectionName: (collectionName: string) => void, - syncWithRemote: () => void, - setWatchFolderServiceIsRunning: (isRunning: boolean) => void, - ) { - try { - this.setElectronFiles = setElectronFiles; - this.setCollectionName = setCollectionName; - this.syncWithRemote = syncWithRemote; - this.setWatchFolderServiceIsRunning = - setWatchFolderServiceIsRunning; - this.setupWatcherFunctions(); - await this.getAndSyncDiffOfFiles(); - } catch (e) { - log.error("error while initializing watch service", e); - } + /** + * Temporarily pause syncing and cancel any running uploads. + * + * This frees up the uploader for handling user initated uploads. + */ + pauseRunningSync() { + this.isPaused = true; + uploadManager.cancelRunningUpload(); } - async getAndSyncDiffOfFiles() { + /** + * Resume from a temporary pause, resyncing from disk. + * + * Sibling of {@link pauseRunningSync}. + */ + resumePausedSync() { + this.isPaused = false; + this.syncWithDisk(); + } + + /** Return the list of folders we are watching for changes. */ + async getWatches(): Promise { + return await ensureElectron().watch.get(); + } + + /** + * Return true if we are currently syncing files that belong to the given + * {@link folderPath}. + */ + isSyncingFolder(folderPath: string) { + return this.activeWatch?.folderPath == folderPath; + } + + /** + * Add a new folder watch for the given root {@link folderPath} + * + * @param mapping The {@link CollectionMapping} to use to decide which + * collection do files belonging to nested directories go to. + * + * @returns The updated list of watches. + */ + async addWatch(folderPath: string, mapping: CollectionMapping) { + const watches = await ensureElectron().watch.add(folderPath, mapping); + this.syncWithDisk(); + return watches; + } + + /** + * Remove the folder watch for the given root {@link folderPath}. + * + * @returns The updated list of watches. + */ + async removeWatch(folderPath: string) { + return await ensureElectron().watch.remove(folderPath); + } + + private async syncWithDisk() { try { - let mappings = await this.getWatchMappings(); - - if (!mappings?.length) { - return; - } - - mappings = await this.filterOutDeletedMappings(mappings); + const watches = await this.getWatches(); + if (!watches) return; this.eventQueue = []; + const events = await deduceEvents(watches); + log.info(`Folder watch deduced ${events.length} events`); + this.eventQueue = this.eventQueue.concat(events); - for (const mapping of mappings) { - const filesOnDisk: ElectronFile[] = - await ensureElectron().getDirFiles(mapping.folderPath); - - this.uploadDiffOfFiles(mapping, filesOnDisk); - this.trashDiffOfFiles(mapping, filesOnDisk); - } + this.debouncedRunNextEvent(); } catch (e) { - log.error("error while getting and syncing diff of files", e); + log.error("Ignoring error while syncing watched folders", e); } } - isMappingSyncInProgress(mapping: WatchMapping) { - return this.currentEvent?.folderPath === mapping.folderPath; - } - - private uploadDiffOfFiles( - mapping: WatchMapping, - filesOnDisk: ElectronFile[], - ) { - const filesToUpload = getValidFilesToUpload(filesOnDisk, mapping); - - if (filesToUpload.length > 0) { - for (const file of filesToUpload) { - const event: EventQueueItem = { - type: "upload", - collectionName: this.getCollectionNameForMapping( - mapping, - file.path, - ), - folderPath: mapping.folderPath, - files: [file], - }; - this.pushEvent(event); - } - } - } - - private trashDiffOfFiles( - mapping: WatchMapping, - filesOnDisk: ElectronFile[], - ) { - const filesToRemove = mapping.syncedFiles.filter((file) => { - return !filesOnDisk.find( - (electronFile) => electronFile.path === file.path, - ); - }); - - if (filesToRemove.length > 0) { - for (const file of filesToRemove) { - const event: EventQueueItem = { - type: "trash", - collectionName: this.getCollectionNameForMapping( - mapping, - file.path, - ), - folderPath: mapping.folderPath, - paths: [file.path], - }; - this.pushEvent(event); - } - } - } - - private async filterOutDeletedMappings( - mappings: WatchMapping[], - ): Promise { - const notDeletedMappings = []; - for (const mapping of mappings) { - const mappingExists = await ensureElectron().isFolder( - mapping.folderPath, - ); - if (!mappingExists) { - ensureElectron().removeWatchMapping(mapping.folderPath); - } else { - notDeletedMappings.push(mapping); - } - } - return notDeletedMappings; - } - - pushEvent(event: EventQueueItem) { + pushEvent(event: WatchEvent) { this.eventQueue.push(event); + log.info("Folder watch event", event); this.debouncedRunNextEvent(); } - async pushTrashedDir(path: string) { - this.trashingDirQueue.push(path); - } + private registerListeners() { + const watch = ensureElectron().watch; - private setupWatcherFunctions() { - ensureElectron().registerWatcherFunctions( - diskFileAddedCallback, - diskFileRemovedCallback, - diskFolderRemovedCallback, - ); - } + // [Note: File renames during folder watch] + // + // Renames come as two file system events - an `onAddFile` + an + // `onRemoveFile` - in an arbitrary order. - async addWatchMapping( - rootFolderName: string, - folderPath: string, - uploadStrategy: UPLOAD_STRATEGY, - ) { - try { - await ensureElectron().addWatchMapping( - rootFolderName, - folderPath, - uploadStrategy, - ); - this.getAndSyncDiffOfFiles(); - } catch (e) { - log.error("error while adding watch mapping", e); - } - } + watch.onAddFile((path: string, watch: FolderWatch) => { + this.pushEvent({ + action: "upload", + collectionName: collectionNameForPath(path, watch), + folderPath: watch.folderPath, + filePath: path, + }); + }); - async mappingsAfterRemovingFolder(folderPath: string) { - await ensureElectron().removeWatchMapping(folderPath); - return await this.getWatchMappings(); - } + watch.onRemoveFile((path: string, watch: FolderWatch) => { + this.pushEvent({ + action: "trash", + collectionName: collectionNameForPath(path, watch), + folderPath: watch.folderPath, + filePath: path, + }); + }); - async getWatchMappings(): Promise { - try { - return (await ensureElectron().getWatchMappings()) ?? []; - } catch (e) { - log.error("error while getting watch mappings", e); - return []; - } - } - - private setIsEventRunning(isEventRunning: boolean) { - this.isEventRunning = isEventRunning; - this.setWatchFolderServiceIsRunning(isEventRunning); + watch.onRemoveDir((path: string, watch: FolderWatch) => { + if (path == watch.folderPath) { + log.info( + `Received file system delete event for a watched folder at ${path}`, + ); + this.deletedFolderPaths.push(path); + } + }); } private async runNextEvent() { - try { - if ( - this.eventQueue.length === 0 || - this.isEventRunning || - this.isPaused - ) { + if (this.eventQueue.length == 0 || this.activeWatch || this.isPaused) + return; + + const skip = (reason: string) => { + log.info(`Ignoring event since ${reason}`); + this.debouncedRunNextEvent(); + }; + + const event = this.dequeueClubbedEvent(); + log.info( + `Processing ${event.action} event for folder watch ${event.folderPath} (collectionName ${event.collectionName}, ${event.filePaths.length} files)`, + ); + + const watch = (await this.getWatches()).find( + (watch) => watch.folderPath == event.folderPath, + ); + if (!watch) { + // Possibly stale + skip(`no folder watch for found for ${event.folderPath}`); + return; + } + + if (event.action === "upload") { + const paths = pathsToUpload(event.filePaths, watch); + if (paths.length == 0) { + skip("none of the files need uploading"); return; } - const event = this.clubSameCollectionEvents(); - log.info( - `running event type:${event.type} collectionName:${event.collectionName} folderPath:${event.folderPath} , fileCount:${event.files?.length} pathsCount: ${event.paths?.length}`, - ); - const mappings = await this.getWatchMappings(); - const mapping = mappings.find( - (mapping) => mapping.folderPath === event.folderPath, - ); - if (!mapping) { - throw Error("no Mapping found for event"); - } - log.info( - `mapping for event rootFolder: ${mapping.rootFolderName} folderPath: ${mapping.folderPath} uploadStrategy: ${mapping.uploadStrategy} syncedFilesCount: ${mapping.syncedFiles.length} ignoredFilesCount ${mapping.ignoredFiles.length}`, - ); - if (event.type === "upload") { - event.files = getValidFilesToUpload(event.files, mapping); - log.info(`valid files count: ${event.files?.length}`); - if (event.files.length === 0) { - return; - } - } - this.currentEvent = event; - this.currentlySyncedMapping = mapping; + // Here we pass control to the uploader. When the upload is done, + // the uploader will notify us by calling allFileUploadsDone. - this.setIsEventRunning(true); - if (event.type === "upload") { - this.processUploadEvent(); - } else { - await this.processTrashEvent(); - this.setIsEventRunning(false); - setTimeout(() => this.runNextEvent(), 0); - } - } catch (e) { - log.error("runNextEvent failed", e); - } - } - - private async processUploadEvent() { - try { + this.activeWatch = watch; this.uploadRunning = true; - this.setCollectionName(this.currentEvent.collectionName); - this.setElectronFiles(this.currentEvent.files); - } catch (e) { - log.error("error while running next upload", e); + const collectionName = event.collectionName; + log.info( + `Folder watch requested upload of ${paths.length} files to collection ${collectionName}`, + ); + + this.upload(collectionName, paths); + } else { + if (this.pruneFileEventsFromDeletedFolderPaths()) { + skip("event was from a deleted folder path"); + return; + } + + const [removed, rest] = watch.syncedFiles.reduce( + ([removed, rest], { path }) => { + (event.filePaths.includes(path) ? rest : removed).push( + watch, + ); + return [removed, rest]; + }, + [[], []], + ); + + this.activeWatch = watch; + + await this.moveToTrash(removed); + + await ensureElectron().watch.updateSyncedFiles( + rest, + watch.folderPath, + ); + + this.activeWatch = undefined; + + this.debouncedRunNextEvent(); } } + /** + * Batch the next run of events with the same action, collection and folder + * path into a single clubbed event that contains the list of all effected + * file paths from the individual events. + */ + private dequeueClubbedEvent(): ClubbedWatchEvent | undefined { + const event = this.eventQueue.shift(); + if (!event) return undefined; + + const filePaths = [event.filePath]; + while ( + this.eventQueue.length > 0 && + event.action === this.eventQueue[0].action && + event.folderPath === this.eventQueue[0].folderPath && + event.collectionName === this.eventQueue[0].collectionName + ) { + filePaths.push(this.eventQueue[0].filePath); + this.eventQueue.shift(); + } + return { ...event, filePaths }; + } + + /** + * Callback invoked by the uploader whenever a file we requested to + * {@link upload} gets uploaded. + */ async onFileUpload( fileUploadResult: UPLOAD_RESULT, fileWithCollection: FileWithCollection, file: EncryptedEnteFile, ) { - log.debug(() => `onFileUpload called`); - if (!this.isUploadRunning()) { - return; - } if ( [ UPLOAD_RESULT.ADDED_SYMLINK, @@ -328,191 +363,151 @@ class WatchFolderService { } } + /** + * Callback invoked by the uploader whenever all the files we requested to + * {@link upload} get uploaded. + */ async allFileUploadsDone( filesWithCollection: FileWithCollection[], collections: Collection[], ) { - try { - log.debug( - () => - `allFileUploadsDone,${JSON.stringify( - filesWithCollection, - )} ${JSON.stringify(collections)}`, + const electron = ensureElectron(); + const watch = this.activeWatch; + + log.debug(() => + JSON.stringify({ + f: "watch/allFileUploadsDone", + filesWithCollection, + collections, + watch, + }), + ); + + const { syncedFiles, ignoredFiles } = + this.parseAllFileUploadsDone(filesWithCollection); + + log.debug(() => + JSON.stringify({ + f: "watch/allFileUploadsDone", + syncedFiles, + ignoredFiles, + }), + ); + + if (syncedFiles.length > 0) + await electron.watch.updateSyncedFiles( + watch.syncedFiles.concat(syncedFiles), + watch.folderPath, ); - const collection = collections.find( - (collection) => - collection.id === filesWithCollection[0].collectionID, + + if (ignoredFiles.length > 0) + await electron.watch.updateIgnoredFiles( + watch.ignoredFiles.concat(ignoredFiles), + watch.folderPath, ); - log.debug(() => `got collection ${!!collection}`); - log.debug( - () => - `${this.isEventRunning} ${this.currentEvent.collectionName} ${collection?.name}`, - ); - if ( - !this.isEventRunning || - this.currentEvent.collectionName !== collection?.name - ) { - return; - } - const syncedFiles: WatchMapping["syncedFiles"] = []; - const ignoredFiles: WatchMapping["ignoredFiles"] = []; - - for (const fileWithCollection of filesWithCollection) { - this.handleUploadedFile( - fileWithCollection, - syncedFiles, - ignoredFiles, - ); - } - - log.debug(() => `syncedFiles ${JSON.stringify(syncedFiles)}`); - log.debug(() => `ignoredFiles ${JSON.stringify(ignoredFiles)}`); - - if (syncedFiles.length > 0) { - this.currentlySyncedMapping.syncedFiles = [ - ...this.currentlySyncedMapping.syncedFiles, - ...syncedFiles, - ]; - await ensureElectron().updateWatchMappingSyncedFiles( - this.currentlySyncedMapping.folderPath, - this.currentlySyncedMapping.syncedFiles, - ); - } - if (ignoredFiles.length > 0) { - this.currentlySyncedMapping.ignoredFiles = [ - ...this.currentlySyncedMapping.ignoredFiles, - ...ignoredFiles, - ]; - await ensureElectron().updateWatchMappingIgnoredFiles( - this.currentlySyncedMapping.folderPath, - this.currentlySyncedMapping.ignoredFiles, - ); - } - - this.runPostUploadsAction(); - } catch (e) { - log.error("error while running all file uploads done", e); - } - } - - private runPostUploadsAction() { - this.setIsEventRunning(false); + this.activeWatch = undefined; this.uploadRunning = false; - this.runNextEvent(); + + this.debouncedRunNextEvent(); } - private handleUploadedFile( - fileWithCollection: FileWithCollection, - syncedFiles: WatchMapping["syncedFiles"], - ignoredFiles: WatchMapping["ignoredFiles"], - ) { - if (fileWithCollection.isLivePhoto) { - const imagePath = ( - fileWithCollection.livePhotoAssets.image as ElectronFile - ).path; - const videoPath = ( - fileWithCollection.livePhotoAssets.video as ElectronFile - ).path; + private parseAllFileUploadsDone(filesWithCollection: FileWithCollection[]) { + const syncedFiles: FolderWatch["syncedFiles"] = []; + const ignoredFiles: FolderWatch["ignoredFiles"] = []; - if ( - this.filePathToUploadedFileIDMap.has(imagePath) && - this.filePathToUploadedFileIDMap.has(videoPath) - ) { - const imageFile = { - path: imagePath, - uploadedFileID: - this.filePathToUploadedFileIDMap.get(imagePath).id, - collectionID: - this.filePathToUploadedFileIDMap.get(imagePath) - .collectionID, - }; - const videoFile = { - path: videoPath, - uploadedFileID: - this.filePathToUploadedFileIDMap.get(videoPath).id, - collectionID: - this.filePathToUploadedFileIDMap.get(videoPath) - .collectionID, - }; - syncedFiles.push(imageFile); - syncedFiles.push(videoFile); - log.debug( - () => - `added image ${JSON.stringify( - imageFile, - )} and video file ${JSON.stringify( - videoFile, - )} to uploadedFiles`, - ); - } else if ( - this.unUploadableFilePaths.has(imagePath) && - this.unUploadableFilePaths.has(videoPath) - ) { - ignoredFiles.push(imagePath); - ignoredFiles.push(videoPath); - log.debug( - () => - `added image ${imagePath} and video file ${videoPath} to rejectedFiles`, - ); - } - this.filePathToUploadedFileIDMap.delete(imagePath); - this.filePathToUploadedFileIDMap.delete(videoPath); - } else { - const filePath = (fileWithCollection.file as ElectronFile).path; + for (const fileWithCollection of filesWithCollection) { + if (fileWithCollection.isLivePhoto) { + const imagePath = ( + fileWithCollection.livePhotoAssets.image as ElectronFile + ).path; + const videoPath = ( + fileWithCollection.livePhotoAssets.video as ElectronFile + ).path; - if (this.filePathToUploadedFileIDMap.has(filePath)) { - const file = { - path: filePath, - uploadedFileID: - this.filePathToUploadedFileIDMap.get(filePath).id, - collectionID: - this.filePathToUploadedFileIDMap.get(filePath) - .collectionID, - }; - syncedFiles.push(file); - log.debug(() => `added file ${JSON.stringify(file)}`); - } else if (this.unUploadableFilePaths.has(filePath)) { - ignoredFiles.push(filePath); - log.debug(() => `added file ${filePath} to rejectedFiles`); + if ( + this.filePathToUploadedFileIDMap.has(imagePath) && + this.filePathToUploadedFileIDMap.has(videoPath) + ) { + const imageFile = { + path: imagePath, + uploadedFileID: + this.filePathToUploadedFileIDMap.get(imagePath).id, + collectionID: + this.filePathToUploadedFileIDMap.get(imagePath) + .collectionID, + }; + const videoFile = { + path: videoPath, + uploadedFileID: + this.filePathToUploadedFileIDMap.get(videoPath).id, + collectionID: + this.filePathToUploadedFileIDMap.get(videoPath) + .collectionID, + }; + syncedFiles.push(imageFile); + syncedFiles.push(videoFile); + log.debug( + () => + `added image ${JSON.stringify( + imageFile, + )} and video file ${JSON.stringify( + videoFile, + )} to uploadedFiles`, + ); + } else if ( + this.unUploadableFilePaths.has(imagePath) && + this.unUploadableFilePaths.has(videoPath) + ) { + ignoredFiles.push(imagePath); + ignoredFiles.push(videoPath); + log.debug( + () => + `added image ${imagePath} and video file ${videoPath} to rejectedFiles`, + ); + } + this.filePathToUploadedFileIDMap.delete(imagePath); + this.filePathToUploadedFileIDMap.delete(videoPath); + } else { + const filePath = (fileWithCollection.file as ElectronFile).path; + + if (this.filePathToUploadedFileIDMap.has(filePath)) { + const file = { + path: filePath, + uploadedFileID: + this.filePathToUploadedFileIDMap.get(filePath).id, + collectionID: + this.filePathToUploadedFileIDMap.get(filePath) + .collectionID, + }; + syncedFiles.push(file); + log.debug(() => `added file ${JSON.stringify(file)}`); + } else if (this.unUploadableFilePaths.has(filePath)) { + ignoredFiles.push(filePath); + log.debug(() => `added file ${filePath} to rejectedFiles`); + } + this.filePathToUploadedFileIDMap.delete(filePath); } - this.filePathToUploadedFileIDMap.delete(filePath); } + + return { syncedFiles, ignoredFiles }; } - private async processTrashEvent() { - try { - if (this.checkAndIgnoreIfFileEventsFromTrashedDir()) { - return; - } + private pruneFileEventsFromDeletedFolderPaths() { + const deletedFolderPath = this.deletedFolderPaths.shift(); + if (!deletedFolderPath) return false; - const { paths } = this.currentEvent; - const filePathsToRemove = new Set(paths); - - const files = this.currentlySyncedMapping.syncedFiles.filter( - (file) => filePathsToRemove.has(file.path), - ); - - await this.trashByIDs(files); - - this.currentlySyncedMapping.syncedFiles = - this.currentlySyncedMapping.syncedFiles.filter( - (file) => !filePathsToRemove.has(file.path), - ); - await ensureElectron().updateWatchMappingSyncedFiles( - this.currentlySyncedMapping.folderPath, - this.currentlySyncedMapping.syncedFiles, - ); - } catch (e) { - log.error("error while running next trash", e); - } + this.eventQueue = this.eventQueue.filter( + (event) => !event.filePath.startsWith(deletedFolderPath), + ); + return true; } - private async trashByIDs(toTrashFiles: WatchMapping["syncedFiles"]) { + private async moveToTrash(syncedFiles: FolderWatch["syncedFiles"]) { try { const files = await getLocalFiles(); - const toTrashFilesMap = new Map(); - for (const file of toTrashFiles) { + const toTrashFilesMap = new Map(); + for (const file of syncedFiles) { toTrashFilesMap.set(file.uploadedFileID, file); } const filesToTrash = files.filter((file) => { @@ -537,204 +532,116 @@ class WatchFolderService { log.error("error while trashing by IDs", e); } } - - private checkAndIgnoreIfFileEventsFromTrashedDir() { - if (this.trashingDirQueue.length !== 0) { - this.ignoreFileEventsFromTrashedDir(this.trashingDirQueue[0]); - this.trashingDirQueue.shift(); - return true; - } - return false; - } - - private ignoreFileEventsFromTrashedDir(trashingDir: string) { - this.eventQueue = this.eventQueue.filter((event) => - event.paths.every((path) => !path.startsWith(trashingDir)), - ); - } - - async getCollectionNameAndFolderPath(filePath: string) { - try { - const mappings = await this.getWatchMappings(); - - const mapping = mappings.find( - (mapping) => - filePath.length > mapping.folderPath.length && - filePath.startsWith(mapping.folderPath) && - filePath[mapping.folderPath.length] === "/", - ); - - if (!mapping) { - throw Error(`no mapping found`); - } - - return { - collectionName: this.getCollectionNameForMapping( - mapping, - filePath, - ), - folderPath: mapping.folderPath, - }; - } catch (e) { - log.error("error while getting collection name", e); - } - } - - private getCollectionNameForMapping( - mapping: WatchMapping, - filePath: string, - ) { - return mapping.uploadStrategy === UPLOAD_STRATEGY.COLLECTION_PER_FOLDER - ? getParentFolderName(filePath) - : mapping.rootFolderName; - } - - async selectFolder(): Promise { - try { - const folderPath = await ensureElectron().selectDirectory(); - return folderPath; - } catch (e) { - log.error("error while selecting folder", e); - } - } - - // Batches all the files to be uploaded (or trashed) from the - // event queue of same collection as the next event - private clubSameCollectionEvents(): EventQueueItem { - const event = this.eventQueue.shift(); - while ( - this.eventQueue.length > 0 && - event.collectionName === this.eventQueue[0].collectionName && - event.type === this.eventQueue[0].type - ) { - if (event.type === "trash") { - event.paths = [...event.paths, ...this.eventQueue[0].paths]; - } else { - event.files = [...event.files, ...this.eventQueue[0].files]; - } - this.eventQueue.shift(); - } - return event; - } - - async isFolder(folderPath: string) { - try { - const isFolder = await ensureElectron().isFolder(folderPath); - return isFolder; - } catch (e) { - log.error("error while checking if folder exists", e); - } - } - - pauseRunningSync() { - this.isPaused = true; - uploadManager.cancelRunningUpload(); - } - - resumePausedSync() { - this.isPaused = false; - this.getAndSyncDiffOfFiles(); - } } -const watchFolderService = new WatchFolderService(); +/** The singleton instance of the {@link FolderWatcher}. */ +const watcher = new FolderWatcher(); -export default watchFolderService; +export default watcher; -const getParentFolderName = (filePath: string) => { - const folderPath = filePath.substring(0, filePath.lastIndexOf("/")); - const folderName = folderPath.substring(folderPath.lastIndexOf("/") + 1); - return folderName; +/** + * A file system watch event encapsulates a change that has occurred on disk + * that needs us to take some action within Ente to synchronize with the user's + * Ente collections. + * + * Events get added in two ways: + * + * - When the app starts, it reads the current state of files on disk and + * compares that with its last known state to determine what all events it + * missed. This is easier than it sounds as we have only two events: add and + * remove. + * + * - When the app is running, it gets live notifications from our file system + * watcher (from the Node.js layer) about changes that have happened on disk, + * which the app then enqueues onto the event queue if they pertain to the + * files we're interested in. + */ +interface WatchEvent { + /** The action to take */ + action: "upload" | "trash"; + /** The path of the root folder corresponding to the {@link FolderWatch}. */ + folderPath: string; + /** The name of the Ente collection the file belongs to. */ + collectionName: string; + /** The absolute path to the file under consideration. */ + filePath: string; +} + +/** + * A composite of multiple {@link WatchEvent}s that only differ in their + * {@link filePath}. + * + * When processing events, we combine a run of events with the same + * {@link action}, {@link folderPath} and {@link collectionName}. This allows us + * to process all the affected {@link filePaths} in one shot. + */ +type ClubbedWatchEvent = Omit & { + filePaths: string[]; }; -async function diskFileAddedCallback(file: ElectronFile) { - try { - const collectionNameAndFolderPath = - await watchFolderService.getCollectionNameAndFolderPath(file.path); +/** + * Determine which events we need to process to synchronize the watched on-disk + * folders to their corresponding collections. + */ +const deduceEvents = async (watches: FolderWatch[]): Promise => { + const electron = ensureElectron(); + const events: WatchEvent[] = []; - if (!collectionNameAndFolderPath) { - return; - } + for (const watch of watches) { + const folderPath = watch.folderPath; - const { collectionName, folderPath } = collectionNameAndFolderPath; + const filePaths = await electron.watch.findFiles(folderPath); - const event: EventQueueItem = { - type: "upload", - collectionName, - folderPath, - files: [file], - }; - watchFolderService.pushEvent(event); - log.info( - `added (upload) to event queue, collectionName:${event.collectionName} folderPath:${event.folderPath}, filesCount: ${event.files.length}`, - ); - } catch (e) { - log.error("error while calling diskFileAddedCallback", e); + // Files that are on disk but not yet synced. + for (const filePath of pathsToUpload(filePaths, watch)) + events.push({ + action: "upload", + folderPath, + collectionName: collectionNameForPath(filePath, watch), + filePath, + }); + + // Previously synced files that are no longer on disk. + for (const filePath of pathsToRemove(filePaths, watch)) + events.push({ + action: "trash", + folderPath, + collectionName: collectionNameForPath(filePath, watch), + filePath, + }); } -} -async function diskFileRemovedCallback(filePath: string) { - try { - const collectionNameAndFolderPath = - await watchFolderService.getCollectionNameAndFolderPath(filePath); + return events; +}; - if (!collectionNameAndFolderPath) { - return; - } +/** + * Filter out hidden files and previously synced or ignored paths from + * {@link paths} to get the list of paths that need to be uploaded to the Ente + * collection. + */ +const pathsToUpload = (paths: string[], watch: FolderWatch) => + paths + // Filter out hidden files (files whose names begins with a dot) + .filter((path) => !isHiddenFile(path)) + // Files that are on disk but not yet synced or ignored. + .filter((path) => !isSyncedOrIgnoredPath(path, watch)); - const { collectionName, folderPath } = collectionNameAndFolderPath; +/** + * Return the paths to previously synced files that are no longer on disk and so + * must be removed from the Ente collection. + */ +const pathsToRemove = (paths: string[], watch: FolderWatch) => + watch.syncedFiles + .map((f) => f.path) + .filter((path) => !paths.includes(path)); - const event: EventQueueItem = { - type: "trash", - collectionName, - folderPath, - paths: [filePath], - }; - watchFolderService.pushEvent(event); - log.info( - `added (trash) to event queue collectionName:${event.collectionName} folderPath:${event.folderPath} , pathsCount: ${event.paths.length}`, - ); - } catch (e) { - log.error("error while calling diskFileRemovedCallback", e); - } -} +const isSyncedOrIgnoredPath = (path: string, watch: FolderWatch) => + watch.ignoredFiles.includes(path) || + watch.syncedFiles.find((f) => f.path === path); -async function diskFolderRemovedCallback(folderPath: string) { - try { - const mappings = await watchFolderService.getWatchMappings(); - const mapping = mappings.find( - (mapping) => mapping.folderPath === folderPath, - ); - if (!mapping) { - log.info(`folder not found in mappings, ${folderPath}`); - throw Error(`Watch mapping not found`); - } - watchFolderService.pushTrashedDir(folderPath); - log.info(`added trashedDir, ${folderPath}`); - } catch (e) { - log.error("error while calling diskFolderRemovedCallback", e); - } -} +const collectionNameForPath = (path: string, watch: FolderWatch) => + watch.collectionMapping == "root" + ? dirname(watch.folderPath) + : parentDirectoryName(path); -export function getValidFilesToUpload( - files: ElectronFile[], - mapping: WatchMapping, -) { - const uniqueFilePaths = new Set(); - return files.filter((file) => { - if (!isSystemFile(file) && !isSyncedOrIgnoredFile(file, mapping)) { - if (!uniqueFilePaths.has(file.path)) { - uniqueFilePaths.add(file.path); - return true; - } - } - return false; - }); -} - -function isSyncedOrIgnoredFile(file: ElectronFile, mapping: WatchMapping) { - return ( - mapping.ignoredFiles.includes(file.path) || - mapping.syncedFiles.find((f) => f.path === file.path) - ); -} +const parentDirectoryName = (path: string) => basename(dirname(path)); diff --git a/web/apps/photos/src/types/upload/index.ts b/web/apps/photos/src/types/upload/index.ts index 3deab0ed7..72eef39f6 100644 --- a/web/apps/photos/src/types/upload/index.ts +++ b/web/apps/photos/src/types/upload/index.ts @@ -156,13 +156,6 @@ export interface ParsedExtractedMetadata { height: number; } -// This is used to prompt the user the make upload strategy choice -export interface ImportSuggestion { - rootFolderName: string; - hasNestedFolders: boolean; - hasRootLevelFileWithFolder: boolean; -} - export interface PublicUploadProps { token: string; passwordToken: string; diff --git a/web/apps/photos/src/types/watchFolder/index.ts b/web/apps/photos/src/types/watchFolder/index.ts deleted file mode 100644 index bd55704de..000000000 --- a/web/apps/photos/src/types/watchFolder/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { UPLOAD_STRATEGY } from "constants/upload"; -import { ElectronFile } from "types/upload"; - -export interface WatchMappingSyncedFile { - path: string; - uploadedFileID: number; - collectionID: number; -} - -export interface WatchMapping { - rootFolderName: string; - folderPath: string; - uploadStrategy: UPLOAD_STRATEGY; - syncedFiles: WatchMappingSyncedFile[]; - ignoredFiles: string[]; -} - -export interface EventQueueItem { - type: "upload" | "trash"; - folderPath: string; - collectionName?: string; - paths?: string[]; - files?: ElectronFile[]; -} diff --git a/web/apps/photos/src/utils/storage/mlIDbStorage.ts b/web/apps/photos/src/utils/storage/mlIDbStorage.ts index 6dccbb89d..40e6dad66 100644 --- a/web/apps/photos/src/utils/storage/mlIDbStorage.ts +++ b/web/apps/photos/src/utils/storage/mlIDbStorage.ts @@ -97,10 +97,8 @@ class MLIDbStorage { wasMLSearchEnabled = searchConfig.enabled; } } catch (e) { - log.info( - "Ignoring likely harmless error while trying to determine ML search status during migration", - e, - ); + // The configs store might not exist (e.g. during logout). + // Ignore. } log.info( `Previous ML database v${oldVersion} had ML search ${wasMLSearchEnabled ? "enabled" : "disabled"}`, diff --git a/web/apps/photos/src/utils/ui/index.tsx b/web/apps/photos/src/utils/ui/index.tsx index 1b01116d3..8f4895ead 100644 --- a/web/apps/photos/src/utils/ui/index.tsx +++ b/web/apps/photos/src/utils/ui/index.tsx @@ -1,5 +1,5 @@ import { ensureElectron } from "@/next/electron"; -import { AppUpdateInfo } from "@/next/types/ipc"; +import { AppUpdate } from "@/next/types/ipc"; import { logoutUser } from "@ente/accounts/services/user"; import { DialogBoxAttributes } from "@ente/shared/components/DialogBox/types"; import AutoAwesomeOutlinedIcon from "@mui/icons-material/AutoAwesomeOutlined"; @@ -55,7 +55,7 @@ export const getTrashFileMessage = (deleteFileHelper): DialogBoxAttributes => ({ export const getUpdateReadyToInstallMessage = ({ version, -}: AppUpdateInfo): DialogBoxAttributes => ({ +}: AppUpdate): DialogBoxAttributes => ({ icon: , title: t("UPDATE_AVAILABLE"), content: t("UPDATE_INSTALLABLE_MESSAGE"), @@ -73,7 +73,7 @@ export const getUpdateReadyToInstallMessage = ({ export const getUpdateAvailableForDownloadMessage = ({ version, -}: AppUpdateInfo): DialogBoxAttributes => ({ +}: AppUpdate): DialogBoxAttributes => ({ icon: , title: t("UPDATE_AVAILABLE"), content: t("UPDATE_AVAILABLE_MESSAGE"), diff --git a/web/apps/photos/src/utils/upload/index.ts b/web/apps/photos/src/utils/upload/index.ts index 643c931fe..4e6d216cf 100644 --- a/web/apps/photos/src/utils/upload/index.ts +++ b/web/apps/photos/src/utils/upload/index.ts @@ -1,18 +1,10 @@ +import { basename, dirname } from "@/next/file"; import { FILE_TYPE } from "constants/file"; -import { - A_SEC_IN_MICROSECONDS, - DEFAULT_IMPORT_SUGGESTION, - PICKED_UPLOAD_TYPE, -} from "constants/upload"; +import { A_SEC_IN_MICROSECONDS, PICKED_UPLOAD_TYPE } from "constants/upload"; import isElectron from "is-electron"; import { exportMetadataDirectoryName } from "services/export"; import { EnteFile } from "types/file"; -import { - ElectronFile, - FileWithCollection, - ImportSuggestion, - Metadata, -} from "types/upload"; +import { ElectronFile, FileWithCollection, Metadata } from "types/upload"; const TYPE_JSON = "json"; const DEDUPE_COLLECTION = new Set(["icloud library", "icloudlibrary"]); @@ -110,15 +102,36 @@ export function areFileWithCollectionsSame( return firstFile.localID === secondFile.localID; } +/** + * Return true if all the paths in the given list are items that belong to the + * same (arbitrary) directory. + * + * Empty list of paths is considered to be in the same directory. + */ +export const areAllInSameDirectory = (paths: string[]) => + new Set(paths.map(dirname)).size == 1; + +// This is used to prompt the user the make upload strategy choice +export interface ImportSuggestion { + rootFolderName: string; + hasNestedFolders: boolean; + hasRootLevelFileWithFolder: boolean; +} + +export const DEFAULT_IMPORT_SUGGESTION: ImportSuggestion = { + rootFolderName: "", + hasNestedFolders: false, + hasRootLevelFileWithFolder: false, +}; + export function getImportSuggestion( uploadType: PICKED_UPLOAD_TYPE, - toUploadFiles: File[] | ElectronFile[], + paths: string[], ): ImportSuggestion { if (isElectron() && uploadType === PICKED_UPLOAD_TYPE.FILES) { return DEFAULT_IMPORT_SUGGESTION; } - const paths: string[] = toUploadFiles.map((file) => file["path"]); const getCharCount = (str: string) => (str.match(/\//g) ?? []).length; paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2)); const firstPath = paths[0]; @@ -209,3 +222,10 @@ export function filterOutSystemFiles(files: File[] | ElectronFile[]) { export function isSystemFile(file: File | ElectronFile) { return file.name.startsWith("."); } + +/** + * Return true if the file at the given {@link path} is hidden. + * + * Hidden files are those whose names begin with a "." (dot). + */ +export const isHiddenFile = (path: string) => basename(path).startsWith("."); diff --git a/web/apps/photos/tests/zip-file-reading.test.ts b/web/apps/photos/tests/zip-file-reading.test.ts index 6ac20bfee..07d70f067 100644 --- a/web/apps/photos/tests/zip-file-reading.test.ts +++ b/web/apps/photos/tests/zip-file-reading.test.ts @@ -96,7 +96,7 @@ export const testZipWithRootFileReadingTest = async () => { const importSuggestion = getImportSuggestion( PICKED_UPLOAD_TYPE.ZIPS, - files, + files.map((file) => file["path"]), ); if (!importSuggestion.rootFolderName) { throw Error( diff --git a/web/packages/next/file.ts b/web/packages/next/file.ts index fae9a6d00..83b20f2ec 100644 --- a/web/packages/next/file.ts +++ b/web/packages/next/file.ts @@ -17,8 +17,12 @@ type FileNameComponents = [name: string, extension: string | undefined]; */ export const nameAndExtension = (fileName: string): FileNameComponents => { const i = fileName.lastIndexOf("."); + // No extension if (i == -1) return [fileName, undefined]; - else return [fileName.slice(0, i), fileName.slice(i + 1)]; + // A hidden file without an extension, e.g. ".gitignore" + if (i == 0) return [fileName, undefined]; + // Both components present, just omit the dot. + return [fileName.slice(0, i), fileName.slice(i + 1)]; }; /** @@ -29,6 +33,39 @@ export const nameAndExtension = (fileName: string): FileNameComponents => { export const fileNameFromComponents = (components: FileNameComponents) => components.filter((x) => !!x).join("."); +/** + * Return the file name portion from the given {@link path}. + * + * This tries to emulate the UNIX `basename` command. In particular, any + * trailing slashes on the path are trimmed, so this function can be used to get + * the name of the directory too. + * + * The path is assumed to use POSIX separators ("/"). + */ +export const basename = (path: string) => { + const pathComponents = path.split("/"); + for (let i = pathComponents.length - 1; i >= 0; i--) + if (pathComponents[i] !== "") return pathComponents[i]; + return path; +}; + +/** + * Return the directory portion from the given {@link path}. + * + * This tries to emulate the UNIX `dirname` command. In particular, any trailing + * slashes on the path are trimmed, so this function can be used to get the path + * leading up to a directory too. + * + * The path is assumed to use POSIX separators ("/"). + */ +export const dirname = (path: string) => { + const pathComponents = path.split("/"); + while (pathComponents.pop() == "") { + /* no-op */ + } + return pathComponents.join("/"); +}; + export function getFileNameSize(file: File | ElectronFile) { return `${file.name}_${convertBytesToHumanReadable(file.size)}`; } diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 85986b639..0628bb0ca 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -5,22 +5,6 @@ import type { ElectronFile } from "./file"; -export interface AppUpdateInfo { - autoUpdatable: boolean; - version: string; -} - -export enum FILE_PATH_TYPE { - FILES = "files", - ZIPS = "zips", -} - -export enum PICKED_UPLOAD_TYPE { - FILES = "files", - FOLDERS = "folders", - ZIPS = "zips", -} - /** * Extra APIs provided by our Node.js layer when our code is running inside our * desktop (Electron) app. @@ -111,7 +95,7 @@ export interface Electron { * Note: Setting a callback clears any previous callbacks. */ onAppUpdateAvailable: ( - cb?: ((updateInfo: AppUpdateInfo) => void) | undefined, + cb?: ((update: AppUpdate) => void) | undefined, ) => void; /** @@ -199,6 +183,12 @@ export interface Electron { * @param contents The string contents to write. */ writeFile: (path: string, contents: string) => Promise; + + /** + * Return true if there is an item at {@link dirPath}, and it is as + * directory. + */ + isDir: (dirPath: string) => Promise; }; /* @@ -284,73 +274,211 @@ export interface Electron { // - Watch - registerWatcherFunctions: ( - addFile: (file: ElectronFile) => Promise, - removeFile: (path: string) => Promise, - removeFolder: (folderPath: string) => Promise, - ) => void; + /** + * Interface with the file system watcher running in our Node.js layer. + * + * [Note: Folder vs Directory in the context of FolderWatch-es] + * + * A note on terminology: The word "folder" is used to the top level root + * folder for which a {@link FolderWatch} has been added. This folder is + * also in 1-1 correspondence to be a directory on the user's disk. It can + * have other, nested directories too (which may or may not be getting + * mapped to separate Ente collections), but we'll not refer to these nested + * directories as folders - only the root of the tree, which the user + * dragged/dropped or selected to set up the folder watch, will be referred + * to as a folder when naming things. + */ + watch: { + /** + * Return the list of folder watches, pruning non-existing directories. + * + * The list of folder paths (and auxillary details) is persisted in the + * Node.js layer. The implementation of this function goes through the + * list, permanently removes any watches whose on-disk directory is no + * longer present, and returns this pruned list of watches. + */ + get: () => Promise; - addWatchMapping: ( - collectionName: string, - folderPath: string, - uploadStrategy: number, - ) => Promise; + /** + * Add a new folder watch for the given {@link folderPath}. + * + * This adds a new entry in the list of watches (persisting them on + * disk), and also starts immediately observing for file system events + * that happen within {@link folderPath}. + * + * @param collectionMapping Determines how nested directories (if any) + * get mapped to Ente collections. + * + * @returns The updated list of watches. + */ + add: ( + folderPath: string, + collectionMapping: CollectionMapping, + ) => Promise; - removeWatchMapping: (folderPath: string) => Promise; + /** + * Remove the pre-existing watch for the given {@link folderPath}. + * + * Persist this removal, and also stop listening for file system events + * that happen within the {@link folderPath}. + * + * @returns The updated list of watches. + */ + remove: (folderPath: string) => Promise; - getWatchMappings: () => Promise; + /** + * Update the list of synced files for the folder watch associated + * with the given {@link folderPath}. + */ + updateSyncedFiles: ( + syncedFiles: FolderWatch["syncedFiles"], + folderPath: string, + ) => Promise; - updateWatchMappingSyncedFiles: ( - folderPath: string, - files: FolderWatch["syncedFiles"], - ) => Promise; + /** + * Update the list of ignored file paths for the folder watch + * associated with the given {@link folderPath}. + */ + updateIgnoredFiles: ( + ignoredFiles: FolderWatch["ignoredFiles"], + folderPath: string, + ) => Promise; - updateWatchMappingIgnoredFiles: ( - folderPath: string, - files: FolderWatch["ignoredFiles"], - ) => Promise; + /** + * Register the function to invoke when a file is added in one of the + * folders we are watching. + * + * The callback function is passed the path to the file that was added, + * and the folder watch it was associated with. + * + * The path is guaranteed to use POSIX separators ('/'). + */ + onAddFile: (f: (path: string, watch: FolderWatch) => void) => void; - // - FS legacy - isFolder: (dirPath: string) => Promise; + /** + * Register the function to invoke when a file is removed in one of the + * folders we are watching. + * + * The callback function is passed the path to the file that was + * removed, and the folder watch it was associated with. + * + * The path is guaranteed to use POSIX separators ('/'). + */ + onRemoveFile: (f: (path: string, watch: FolderWatch) => void) => void; + + /** + * Register the function to invoke when a directory is removed in one of + * the folders we are watching. + * + * The callback function is passed the path to the directory that was + * removed, and the folder watch it was associated with. + * + * The path is guaranteed to use POSIX separators ('/'). + */ + onRemoveDir: (f: (path: string, watch: FolderWatch) => void) => void; + + /** + * Return the paths of all the files under the given folder. + * + * This function walks the directory tree starting at {@link folderPath} + * and returns a list of the absolute paths of all the files that exist + * therein. It will recursively traverse into nested directories, and + * return the absolute paths of the files there too. + * + * The returned paths are guaranteed to use POSIX separators ('/'). + */ + findFiles: (folderPath: string) => Promise; + }; // - Upload - getPendingUploads: () => Promise<{ - files: ElectronFile[]; - collectionName: string; - type: string; - }>; - setToUploadFiles: ( - /** TODO(MR): This is the actual type */ - // type: FILE_PATH_TYPE, - type: PICKED_UPLOAD_TYPE, + /** + * Return any pending uploads that were previously enqueued but haven't yet + * been completed. + * + * The state of pending uploads is persisted in the Node.js layer. + * + * Note that we might have both outstanding zip and regular file uploads at + * the same time. In such cases, the zip file ones get precedence. + */ + pendingUploads: () => Promise; + + /** + * Set or clear the name of the collection where the pending upload is + * directed to. + */ + setPendingUploadCollection: (collectionName: string) => Promise; + + /** + * Update the list of files (of {@link type}) associated with the pending + * upload. + */ + setPendingUploadFiles: ( + type: PendingUploads["type"], filePaths: string[], ) => Promise; + + // - + getElectronFilesFromGoogleZip: ( filePath: string, ) => Promise; - setToUploadCollection: (collectionName: string) => Promise; getDirFiles: (dirPath: string) => Promise; } +/** + * Data passed across the IPC bridge when an app update is available. + */ +export interface AppUpdate { + /** `true` if the user automatically update to this (new) version */ + autoUpdatable: boolean; + /** The new version that is available */ + version: string; +} + /** * A top level folder that was selected by the user for watching. * * The user can set up multiple such watches. Each of these can in turn be - * syncing multiple on disk folders to one or more (dependening on the - * {@link uploadStrategy}) Ente albums. + * syncing multiple on disk folders to one or more Ente collections (depending + * on the value of {@link collectionMapping}). * * This type is passed across the IPC boundary. It is persisted on the Node.js * side. */ export interface FolderWatch { - rootFolderName: string; - uploadStrategy: number; + /** + * Specify if nested files should all be mapped to the same single root + * collection, or if there should be a collection per directory that has + * files. @see {@link CollectionMapping}. + */ + collectionMapping: CollectionMapping; + /** + * The path to the (root) folder we are watching. + */ folderPath: string; + /** + * Files that have already been uploaded. + */ syncedFiles: FolderWatchSyncedFile[]; + /** + * Files (paths) that should be ignored when uploading. + */ ignoredFiles: string[]; } +/** + * The ways in which directories are mapped to collection. + * + * This comes into play when we have nested directories that we are trying to + * upload or watch on the user's local file system. + */ +export type CollectionMapping = + /** All files go into a single collection named after the root directory. */ + | "root" + /** Each file goes to a collection named after its parent directory. */ + | "parent"; + /** * An on-disk file that was synced as part of a folder watch. */ @@ -359,3 +487,16 @@ export interface FolderWatchSyncedFile { uploadedFileID: number; collectionID: number; } + +/** + * When the user starts an upload, we remember the files they'd selected or drag + * and dropped so that we can resume (if needed) when the app restarts after + * being stopped in the middle of the uploads. + */ +export interface PendingUploads { + /** The collection to which we're uploading */ + collectionName: string; + /* The upload can be either of a Google Takeout zip, or regular files */ + type: "files" | "zips"; + files: ElectronFile[]; +} diff --git a/web/packages/utils/ensure.ts b/web/packages/utils/ensure.ts new file mode 100644 index 000000000..2e8f9a213 --- /dev/null +++ b/web/packages/utils/ensure.ts @@ -0,0 +1,7 @@ +/** + * Throw an exception if the given value is undefined. + */ +export const ensure = (v: T | undefined): T => { + if (v === undefined) throw new Error("Required value was not found"); + return v; +};