WIP 2

desktop/src/main/ipc.ts

@@ -20,7 +20,6 @@ import {
 import {
     fsExists,
     fsIsDir,
-    fsListFiles,
     fsMkdirIfNeeded,
     fsReadTextFile,
     fsRename,
@@ -56,6 +55,7 @@ import {
 } from "./services/upload";
 import {
     addWatchMapping,
+    findFiles,
     getWatchMappings,
     removeWatchMapping,
     updateWatchMappingIgnoredFiles,
@@ -135,8 +135,6 @@ export const attachIPCHandlers = () => {
 
     ipcMain.handle("fsIsDir", (_, dirPath: string) => fsIsDir(dirPath));
 
-    ipcMain.handle("fsListFiles", (_, dirPath: string) => fsListFiles(dirPath));
-
     // - Conversion
 
     ipcMain.handle("convertToJPEG", (_, fileData, filename) =>
@@ -219,6 +217,10 @@ export const attachIPCHandlers = () => {
 export const attachFSWatchIPCHandlers = (watcher: FSWatcher) => {
     // - Watch
 
+    ipcMain.handle("findFiles", (_, folderPath: string) =>
+        findFiles(folderPath),
+    );
+
     ipcMain.handle(
         "addWatchMapping",
         (

desktop/src/main/services/watch.ts

@@ -1,33 +1,23 @@
 import type { FSWatcher } from "chokidar";
 import ElectronLog from "electron-log";
+import fs from "node:fs/promises";
+import path from "node:path";
 import { FolderWatch, WatchStoreType } from "../../types/ipc";
 import { watchStore } from "../stores/watch.store";
-import path from "node:path";
-import fs from "node:fs/promises";
 
-/**
- * Return the paths of all the files under the given directory (recursive).
- *
- * This function walks the directory tree starting at {@link dirPath}, and
- * returns a list of the absolute paths of all the files that exist therein. It
- * will recursively traverse into nested directories, and return the absolute
- * paths of the files there too.
- *
- * The returned paths are guaranteed to use POSIX separators ('/').
- */
 export const findFiles = async (dirPath: string) => {
-    const items = await fs.readdir(dirPath, { withFileTypes: true })
+    const items = await fs.readdir(dirPath, { withFileTypes: true });
     let paths: string[] = [];
     for (const item of items) {
         const itemPath = path.posix.join(dirPath, item.name);
         if (item.isFile()) {
-            paths.push(itemPath)
+            paths.push(itemPath);
         } else if (item.isDirectory()) {
-            paths = [...paths, ...await findFiles(itemPath)]
+            paths = [...paths, ...(await findFiles(itemPath))];
         }
     }
-    return paths
-}
+    return paths;
+};
 
 export const addWatchMapping = async (
     watcher: FSWatcher,

desktop/src/preload.ts

@@ -121,9 +121,6 @@ const fsWriteFile = (path: string, contents: string): Promise<void> =>
 const fsIsDir = (dirPath: string): Promise<boolean> =>
     ipcRenderer.invoke("fsIsDir", dirPath);
 
-const fsListFiles = (dirPath: string): Promise<string[]> =>
-    ipcRenderer.invoke("fsListFiles", dirPath);
-
 // - AUDIT below this
 
 // - Conversion
@@ -194,6 +191,9 @@ const showUploadZipDialog = (): Promise<{
 
 // - Watch
 
+const findFiles = (folderPath: string): Promise<string[]> =>
+    ipcRenderer.invoke("findFiles", folderPath);
+
 const registerWatcherFunctions = (
     addFile: (file: ElectronFile) => Promise<void>,
     removeFile: (path: string) => Promise<void>,
@@ -325,7 +325,6 @@ contextBridge.exposeInMainWorld("electron", {
         readTextFile: fsReadTextFile,
         writeFile: fsWriteFile,
         isDir: fsIsDir,
-        listFiles: fsListFiles,
     },
 
     // - Conversion
@@ -346,6 +345,9 @@ contextBridge.exposeInMainWorld("electron", {
     showUploadZipDialog,
 
     // - Watch
+    watch: {
+        findFiles,
+    },
     registerWatcherFunctions,
     addWatchMapping,
     removeWatchMapping,

web/apps/photos/src/services/watch.ts

@@ -4,6 +4,7 @@
  */
 
 import { ensureElectron } from "@/next/electron";
+import { nameAndExtension } from "@/next/file";
 import log from "@/next/log";
 import type { FolderWatch } from "@/next/types/ipc";
 import { UPLOAD_RESULT, UPLOAD_STRATEGY } from "constants/upload";
@@ -19,15 +20,15 @@ import { removeFromCollection } from "./collectionService";
 import { getLocalFiles } from "./fileService";
 
 /**
- * A file system event encapsulates a change that has occurred on disk that
- * needs us to take some action within Ente to synchronize with the user's
- * (Ente) albums.
+ * A file system watch event encapsulates a change that has occurred on disk
+ * that needs us to take some action within Ente to synchronize with the user's
+ * Ente collections.
  *
  * Events get added in two ways:
  *
  * - When the app starts, it reads the current state of files on disk and
  *   compares that with its last known state to determine what all events it
- *   missed. This is easier than it sounds as we have only two events, add and
+ *   missed. This is easier than it sounds as we have only two events: add and
  *   remove.
  *
  * - When the app is running, it gets live notifications from our file system
@@ -35,20 +36,20 @@ import { getLocalFiles } from "./fileService";
  *   which the app then enqueues onto the event queue if they pertain to the
  *   files we're interested in.
  */
-interface FSEvent {
+interface WatchEvent {
     /** The action to take */
     action: "upload" | "trash";
     /** The path of the root folder corresponding to the {@link FolderWatch}. */
     folderPath: string;
-    /** If applicable, the name of the (Ente) collection the file belongs to. */
+    /** The name of the Ente collection the file belongs to. */
     collectionName?: string;
     /** The absolute path to the file under consideration. */
     filePath: string;
 }
 
 class WatchFolderService {
-    private eventQueue: FSEvent[] = [];
-    private currentEvent: FSEvent;
+    private eventQueue: WatchEvent[] = [];
+    private currentEvent: WatchEvent;
     private currentlySyncedMapping: WatchMapping;
     private trashingDirQueue: string[] = [];
     private isEventRunning: boolean = false;
@@ -87,26 +88,26 @@ class WatchFolderService {
             this.setWatchFolderServiceIsRunning =
                 setWatchFolderServiceIsRunning;
             this.setupWatcherFunctions();
-            await this.getAndSyncDiffOfFiles();
+            await this.syncWithDisk();
         } catch (e) {
             log.error("error while initializing watch service", e);
         }
     }
 
-    async getAndSyncDiffOfFiles() {
+    private async syncWithDisk() {
         try {
             const electron = ensureElectron();
             const mappings = await electron.getWatchMappings();
             if (!mappings) return;
 
             this.eventQueue = [];
-            const { events, nonExistentFolderPaths } =
-                await syncWithDisk(mappings);
+            const { events, deletedFolderPaths } = await deduceEvents(mappings);
             this.eventQueue = [...this.eventQueue, ...events];
-            this.debouncedRunNextEvent();
 
-            for (const path of nonExistentFolderPaths)
+            for (const path of deletedFolderPaths)
                 electron.removeWatchMapping(path);
+
+            this.debouncedRunNextEvent();
         } catch (e) {
             log.error("Ignoring error while syncing watched folders", e);
         }
@@ -116,9 +117,9 @@ class WatchFolderService {
         return this.currentEvent?.folderPath === mapping.folderPath;
     }
 
-    pushEvent(event: EventQueueItem) {
+    private pushEvent(event: WatchEvent) {
         this.eventQueue.push(event);
-        log.info("FS event", event);
+        log.info("Watch event", event);
         this.debouncedRunNextEvent();
     }
 
@@ -145,7 +146,7 @@ class WatchFolderService {
                 folderPath,
                 uploadStrategy,
             );
-            this.getAndSyncDiffOfFiles();
+            this.syncWithDisk();
         } catch (e) {
             log.error("error while adding watch mapping", e);
         }
@@ -576,7 +577,7 @@ class WatchFolderService {
 
     resumePausedSync() {
         this.isPaused = false;
-        this.getAndSyncDiffOfFiles();
+        this.syncWithDisk();
     }
 }
 
@@ -584,12 +585,6 @@ const watchFolderService = new WatchFolderService();
 
 export default watchFolderService;
 
-const getParentFolderName = (filePath: string) => {
-    const folderPath = filePath.substring(0, filePath.lastIndexOf("/"));
-    const folderName = folderPath.substring(folderPath.lastIndexOf("/") + 1);
-    return folderName;
-};
-
 async function diskFileAddedCallback(file: ElectronFile) {
     const collectionNameAndFolderPath =
         await watchFolderService.getCollectionNameAndFolderPath(file.path);
@@ -669,37 +664,35 @@ function isSyncedOrIgnoredFile(file: ElectronFile, mapping: WatchMapping) {
 }
 
 /**
- * Determine which events we need to process to synchronize the watched albums
- * with the corresponding on disk folders.
+ * Determine which events we need to process to synchronize the watched on-disk
+ * folders to their corresponding collections.
  *
- * Also return a list of previously created folder watches for this there is no
- * longer any no corresponding folder on disk.
+ * Also return a list of previously created folder watches for which there is no
+ * longer any no corresponding directory on disk.
  */
-const syncWithDisk = async (
+const deduceEvents = async (
     mappings: FolderWatch[],
 ): Promise<{
-    events: EventQueueItem[];
-    nonExistentFolderPaths: string[];
+    events: WatchEvent[];
+    deletedFolderPaths: string[];
 }> => {
     const activeMappings = [];
-    const nonExistentFolderPaths: string[] = [];
+    const deletedFolderPaths: string[] = [];
 
     for (const mapping of mappings) {
         const valid = await electron.fs.isDir(mapping.folderPath);
-        if (!valid) nonExistentFolderPaths.push(mapping.folderPath);
+        if (!valid) deletedFolderPaths.push(mapping.folderPath);
         else activeMappings.push(mapping);
     }
 
-    const events: EventQueueItem[] = [];
+    const events: WatchEvent[] = [];
 
     for (const mapping of activeMappings) {
         const folderPath = mapping.folderPath;
 
-        const paths = (await electron.fs.listFiles(folderPath))
+        const paths = (await electron.watch.findFiles(folderPath))
             // Filter out hidden files (files whose names begins with a dot)
-            .filter((n) => !n.startsWith("."))
-            // Prepend folderPath to get the full path
-            .map((f) => `${folderPath}/${f}`);
+            .filter((path) => !nameAndExtension(path)[0].startsWith("."));
 
         // Files that are on disk but not yet synced.
         const pathsToUpload = paths.filter(
@@ -708,7 +701,7 @@ const syncWithDisk = async (
 
         for (const path of pathsToUpload)
             events.push({
-                type: "upload",
+                action: "upload",
                 collectionName: getCollectionNameForMapping(mapping, path),
                 folderPath,
                 filePath: path,
@@ -728,7 +721,7 @@ const syncWithDisk = async (
             });
     }
 
-    return { events, nonExistentFolderPaths };
+    return { events, deletedFolderPaths };
 };
 
 function isSyncedOrIgnoredPath(path: string, mapping: WatchMapping) {
@@ -743,6 +736,16 @@ const getCollectionNameForMapping = (
     filePath: string,
 ) => {
     return mapping.uploadStrategy === UPLOAD_STRATEGY.COLLECTION_PER_FOLDER
-        ? getParentFolderName(filePath)
+        ? parentDirectoryName(filePath)
         : mapping.rootFolderName;
 };
+
+const parentDirectoryName = (filePath: string) => {
+    const components = filePath.split("/");
+    const parentName = components[components.length - 2];
+    if (!parentName)
+        throw new Error(
+            `Unexpected file path without a parent folder: ${filePath}`,
+        );
+    return parentName;
+};

web/packages/next/types/ipc.ts

@@ -205,18 +205,6 @@ export interface Electron {
          * directory.
          */
         isDir: (dirPath: string) => Promise<boolean>;
-
-        /**
-         * Return a list of the file names of the files in the given directory.
-         *
-         * Note:
-         *
-         * - This is not recursive, it will only return the names of direct
-         *   children.
-         *
-         * - It will return only the names of files, not directories.
-         */
-        listFiles: (dirPath: string) => Promise<string[]>;
     };
 
     /*
@@ -303,15 +291,32 @@ export interface Electron {
     // - Watch
 
     /**
-     * Get the latest state of the watched folders.
+     * Functions tailored for the folder watch functionality
      *
-     * We persist the folder watches that the user has setup. This function goes
-     * through that list, prunes any folders that don't exist on disk anymore,
-     * and for each, also returns a list of files that exist in that folder.
+     * [Note: Folder vs Directory in the context of FolderWatch-es]
+     *
+     * A note on terminology: The word "folder" is used to the top level root
+     * folder for which a {@link FolderWatch} has been added. This folder is
+     * also in 1-1 correspondence to be a directory on the user's disk. It can
+     * have other, nested directories too (which may or may not be getting
+     * mapped to separate Ente collections), but we'll not refer to these nested
+     * directories as folders - only the root of the tree, which the user
+     * dragged/dropped or selected to set up the folder watch, will be referred
+     * to as a folder when naming things.
      */
-    folderWatchesAndFilesTherein: () => Promise<
-        [watch: FolderWatch, files: ElectronFile[]][]
-    >;
+    watch: {
+        /**
+         * Return the paths of all the files under the given folder.
+         *
+         * This function walks the directory tree starting at {@link folderPath}
+         * and returns a list of the absolute paths of all the files that exist
+         * therein. It will recursively traverse into nested directories, and
+         * return the absolute paths of the files there too.
+         *
+         * The returned paths are guaranteed to use POSIX separators ('/').
+         */
+        findFiles: (folderPath: string) => Promise<string[]>;
+    };
 
     registerWatcherFunctions: (
         addFile: (file: ElectronFile) => Promise<void>,
@@ -327,6 +332,15 @@ export interface Electron {
 
     removeWatchMapping: (folderPath: string) => Promise<void>;
 
+    /**
+     * TODO(MR): Outdated description
+     * Get the latest state of the watched folders.
+     *
+     * We persist the folder watches that the user has setup. This function goes
+     * through that list, prunes any folders that don't exist on disk anymore,
+     * and for each, also returns a list of files that exist in that folder.
+     */
+
     getWatchMappings: () => Promise<FolderWatch[]>;
 
     updateWatchMappingSyncedFiles: (