diff --git a/desktop/src/main/fs.ts b/desktop/src/main/fs.ts index ebf53487c..2428d3a80 100644 --- a/desktop/src/main/fs.ts +++ b/desktop/src/main/fs.ts @@ -27,8 +27,3 @@ export const fsIsDir = async (dirPath: string) => { const stat = await fs.stat(dirPath); return stat.isDirectory(); }; - -export const fsLsFiles = async (dirPath: string) => - (await fs.readdir(dirPath, { withFileTypes: true })) - .filter((e) => e.isFile()) - .map((e) => e.name); diff --git a/desktop/src/main/ipc.ts b/desktop/src/main/ipc.ts index 2e01b4a10..555e51ab1 100644 --- a/desktop/src/main/ipc.ts +++ b/desktop/src/main/ipc.ts @@ -20,7 +20,7 @@ import { import { fsExists, fsIsDir, - fsLsFiles, + fsListFiles, fsMkdirIfNeeded, fsReadTextFile, fsRename, @@ -135,7 +135,7 @@ export const attachIPCHandlers = () => { ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath)); - ipcMain.handle("fsLsFiles", (_, dirPath: string) => fsLsFiles(dirPath)); + ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath)); // - Conversion diff --git a/desktop/src/main/services/watch.ts b/desktop/src/main/services/watch.ts index 1d466d415..c8cd5ec82 100644 --- a/desktop/src/main/services/watch.ts +++ b/desktop/src/main/services/watch.ts @@ -2,6 +2,32 @@ import type { FSWatcher } from "chokidar"; import ElectronLog from "electron-log"; import { FolderWatch, WatchStoreType } from "../../types/ipc"; import { watchStore } from "../stores/watch.store"; +import path from "node:path"; +import fs from "node:fs/promises"; + +/** + * Return the paths of all the files under the given directory (recursive). + * + * This function walks the directory tree starting at {@link dirPath}, and + * returns a list of the absolute paths of all the files that exist therein. It + * will recursively traverse into nested directories, and return the absolute + * paths of the files there too. + * + * The returned paths are guaranteed to use POSIX separators ('/'). + */ +export const findFiles = async (dirPath: string) => { + const items = await fs.readdir(dirPath, { withFileTypes: true }) + let paths: string[] = []; + for (const item of items) { + const itemPath = path.posix.join(dirPath, item.name); + if (item.isFile()) { + paths.push(itemPath) + } else if (item.isDirectory()) { + paths = [...paths, ...await findFiles(itemPath)] + } + } + return paths +} export const addWatchMapping = async ( watcher: FSWatcher, diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 2c2677cf1..0a55e5700 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -121,8 +121,8 @@ const fsWriteFile = (path: string, contents: string): Promise => const fsIsDir = (dirPath: string): Promise => ipcRenderer.invoke("fsIsDir", dirPath); -const fsLsFiles = (dirPath: string): Promise => - ipcRenderer.invoke("fsLsFiles", dirPath); +const fsListFiles = (dirPath: string): Promise => + ipcRenderer.invoke("fsListFiles", dirPath); // - AUDIT below this @@ -325,7 +325,7 @@ contextBridge.exposeInMainWorld("electron", { readTextFile: fsReadTextFile, writeFile: fsWriteFile, isDir: fsIsDir, - lsFiles: fsLsFiles, + listFiles: fsListFiles, }, // - Conversion diff --git a/web/apps/photos/src/services/watch.ts b/web/apps/photos/src/services/watch.ts index bb11fa67e..be4f95aa0 100644 --- a/web/apps/photos/src/services/watch.ts +++ b/web/apps/photos/src/services/watch.ts @@ -12,19 +12,43 @@ import uploadManager from "services/upload/uploadManager"; import { Collection } from "types/collection"; import { EncryptedEnteFile } from "types/file"; import { ElectronFile, FileWithCollection } from "types/upload"; -import { - EventQueueItem, - WatchMapping, - WatchMappingSyncedFile, -} from "types/watchFolder"; +import { WatchMapping, WatchMappingSyncedFile } from "types/watchFolder"; import { groupFilesBasedOnCollectionID } from "utils/file"; import { isSystemFile } from "utils/upload"; import { removeFromCollection } from "./collectionService"; import { getLocalFiles } from "./fileService"; +/** + * A file system event encapsulates a change that has occurred on disk that + * needs us to take some action within Ente to synchronize with the user's + * (Ente) albums. + * + * Events get added in two ways: + * + * - When the app starts, it reads the current state of files on disk and + * compares that with its last known state to determine what all events it + * missed. This is easier than it sounds as we have only two events, add and + * remove. + * + * - When the app is running, it gets live notifications from our file system + * watcher (from the Node.js layer) about changes that have happened on disk, + * which the app then enqueues onto the event queue if they pertain to the + * files we're interested in. + */ +interface FSEvent { + /** The action to take */ + action: "upload" | "trash"; + /** The path of the root folder corresponding to the {@link FolderWatch}. */ + folderPath: string; + /** If applicable, the name of the (Ente) collection the file belongs to. */ + collectionName?: string; + /** The absolute path to the file under consideration. */ + filePath: string; +} + class WatchFolderService { - private eventQueue: EventQueueItem[] = []; - private currentEvent: EventQueueItem; + private eventQueue: FSEvent[] = []; + private currentEvent: FSEvent; private currentlySyncedMapping: WatchMapping; private trashingDirQueue: string[] = []; private isEventRunning: boolean = false; @@ -94,6 +118,7 @@ class WatchFolderService { pushEvent(event: EventQueueItem) { this.eventQueue.push(event); + log.info("FS event", event); this.debouncedRunNextEvent(); } @@ -566,55 +591,41 @@ const getParentFolderName = (filePath: string) => { }; async function diskFileAddedCallback(file: ElectronFile) { - try { - const collectionNameAndFolderPath = - await watchFolderService.getCollectionNameAndFolderPath(file.path); + const collectionNameAndFolderPath = + await watchFolderService.getCollectionNameAndFolderPath(file.path); - if (!collectionNameAndFolderPath) { - return; - } - - const { collectionName, folderPath } = collectionNameAndFolderPath; - - const event: EventQueueItem = { - type: "upload", - collectionName, - folderPath, - files: [file], - }; - watchFolderService.pushEvent(event); - log.info( - `added (upload) to event queue, collectionName:${event.collectionName} folderPath:${event.folderPath}, filesCount: ${event.files.length}`, - ); - } catch (e) { - log.error("error while calling diskFileAddedCallback", e); + if (!collectionNameAndFolderPath) { + return; } + + const { collectionName, folderPath } = collectionNameAndFolderPath; + + const event: EventQueueItem = { + type: "upload", + collectionName, + folderPath, + path: file.path, + }; + watchFolderService.pushEvent(event); } async function diskFileRemovedCallback(filePath: string) { - try { - const collectionNameAndFolderPath = - await watchFolderService.getCollectionNameAndFolderPath(filePath); + const collectionNameAndFolderPath = + await watchFolderService.getCollectionNameAndFolderPath(filePath); - if (!collectionNameAndFolderPath) { - return; - } - - const { collectionName, folderPath } = collectionNameAndFolderPath; - - const event: EventQueueItem = { - type: "trash", - collectionName, - folderPath, - paths: [filePath], - }; - watchFolderService.pushEvent(event); - log.info( - `added (trash) to event queue collectionName:${event.collectionName} folderPath:${event.folderPath} , pathsCount: ${event.paths.length}`, - ); - } catch (e) { - log.error("error while calling diskFileRemovedCallback", e); + if (!collectionNameAndFolderPath) { + return; } + + const { collectionName, folderPath } = collectionNameAndFolderPath; + + const event: EventQueueItem = { + type: "trash", + collectionName, + folderPath, + path: filePath, + }; + watchFolderService.pushEvent(event); } async function diskFolderRemovedCallback(folderPath: string) { @@ -682,34 +693,51 @@ const syncWithDisk = async ( const events: EventQueueItem[] = []; for (const mapping of activeMappings) { - const files = await electron.getDirFiles(mapping.folderPath); + const folderPath = mapping.folderPath; - const filesToUpload = getValidFilesToUpload(files, mapping); + const paths = (await electron.fs.listFiles(folderPath)) + // Filter out hidden files (files whose names begins with a dot) + .filter((n) => !n.startsWith(".")) + // Prepend folderPath to get the full path + .map((f) => `${folderPath}/${f}`); - for (const file of filesToUpload) + // Files that are on disk but not yet synced. + const pathsToUpload = paths.filter( + (path) => !isSyncedOrIgnoredPath(path, mapping), + ); + + for (const path of pathsToUpload) events.push({ type: "upload", - collectionName: getCollectionNameForMapping(mapping, file.path), - folderPath: mapping.folderPath, - files: [file], + collectionName: getCollectionNameForMapping(mapping, path), + folderPath, + filePath: path, }); - const filesToRemove = mapping.syncedFiles.filter((file) => { - return !files.find((f) => f.path === file.path); - }); + // Synced files that are no longer on disk + const pathsToRemove = mapping.syncedFiles.filter( + (file) => !paths.includes(file.path), + ); - for (const file of filesToRemove) + for (const path of pathsToRemove) events.push({ type: "trash", - collectionName: getCollectionNameForMapping(mapping, file.path), + collectionName: getCollectionNameForMapping(mapping, path), folderPath: mapping.folderPath, - paths: [file.path], + filePath: path, }); } return { events, nonExistentFolderPaths }; }; +function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) { + return ( + mapping.ignoredFiles.includes(path) || + mapping.syncedFiles.find((f) => f.path === path) + ); +} + const getCollectionNameForMapping = ( mapping: WatchMapping, filePath: string, diff --git a/web/apps/photos/src/types/watchFolder/index.ts b/web/apps/photos/src/types/watchFolder/index.ts index bd55704de..dda243e55 100644 --- a/web/apps/photos/src/types/watchFolder/index.ts +++ b/web/apps/photos/src/types/watchFolder/index.ts @@ -1,5 +1,4 @@ import { UPLOAD_STRATEGY } from "constants/upload"; -import { ElectronFile } from "types/upload"; export interface WatchMappingSyncedFile { path: string; @@ -14,11 +13,3 @@ export interface WatchMapping { syncedFiles: WatchMappingSyncedFile[]; ignoredFiles: string[]; } - -export interface EventQueueItem { - type: "upload" | "trash"; - folderPath: string; - collectionName?: string; - paths?: string[]; - files?: ElectronFile[]; -} diff --git a/web/packages/next/types/ipc.ts b/web/packages/next/types/ipc.ts index 55e7fca3f..b14c8e270 100644 --- a/web/packages/next/types/ipc.ts +++ b/web/packages/next/types/ipc.ts @@ -207,7 +207,7 @@ export interface Electron { isDir: (dirPath: string) => Promise; /** - * Return a list of the names of the files in the given directory. + * Return a list of the file names of the files in the given directory. * * Note: * @@ -216,7 +216,7 @@ export interface Electron { * * - It will return only the names of files, not directories. */ - lsFiles: (dirPath: string) => Promise; + listFiles: (dirPath: string) => Promise; }; /*