Merge branch 'main' into cast

.gitignore

@@ -39,7 +39,6 @@ out_functions
 out_publish
 
 .env
-.vscode/
 .idea/
 
 # workbox

.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+    "recommendations": ["dbaeumer.vscode-eslint", "esbenp.prettier-vscode"]
+}

.vscode/settings.json

@@ -0,0 +1,17 @@
+{
+    "eslint.validate": [
+        "typescript",
+        "typescriptreact",
+        "javascript",
+        "javascriptreact"
+    ],
+    "prettier.singleQuote": true,
+    "typescript.tsdk": "node_modules/typescript/lib",
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.formatOnPaste": true,
+    "editor.codeActionsOnSave": {
+        "source.fixAll": true
+    },
+    "typescript.enablePromptUseWorkspaceTsdk": true
+}

README.md

@@ -40,7 +40,7 @@ The deployed application is accessible @ [web.ente.io](https://web.ente.io).
 1. Clone this repository with `git clone https://github.com/ente-io/photos-web.git`
 2. Pull in all submodules with `git submodule update --init --recursive`
 3. Install dependencies with `yarn install`
-4. Finally, run the development server with `yarn dev`
+4. Finally, run the development server with `yarn dev:photos`
 
 Open [http://localhost:3000](http://localhost:3000) on your browser to see the live application.
 

apps/auth/public/locales/de/translation.json

@@ -83,9 +83,9 @@
     "ZOOM_IN_OUT": "Herein-/Herauszoomen",
     "PREVIOUS": "",
     "NEXT": "",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "Ente Photos",
+    "TITLE_ALBUMS": "Ente Photos",
+    "TITLE_AUTH": "Ente Auth",
     "UPLOAD_FIRST_PHOTO": "Lade dein erstes Foto hoch",
     "IMPORT_YOUR_FOLDERS": "Importiere deiner Ordner",
     "UPLOAD_DROPZONE_MESSAGE": "",

apps/auth/public/locales/en/translation.json

@@ -83,9 +83,9 @@
     "ZOOM_IN_OUT": "Zoom in/out",
     "PREVIOUS": "Previous (←)",
     "NEXT": "Next (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "Ente Photos",
+    "TITLE_ALBUMS": "Ente Photos",
+    "TITLE_AUTH": "Ente Auth",
     "UPLOAD_FIRST_PHOTO": "Upload your first photo",
     "IMPORT_YOUR_FOLDERS": "Import your folders",
     "UPLOAD_DROPZONE_MESSAGE": "Drop to backup your files",

apps/auth/public/locales/fr/translation.json

@@ -83,9 +83,9 @@
     "ZOOM_IN_OUT": "Zoom +/-",
     "PREVIOUS": "Précédent (←)",
     "NEXT": "Suivant (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "Ente Photos",
+    "TITLE_ALBUMS": "Ente Photos",
+    "TITLE_AUTH": "Ente Auth",
     "UPLOAD_FIRST_PHOTO": "Chargez votre 1ere photo",
     "IMPORT_YOUR_FOLDERS": "Importez vos dossiers",
     "UPLOAD_DROPZONE_MESSAGE": "Déposez pour sauvegarder vos fichiers",

apps/auth/public/locales/it/translation.json

@@ -83,9 +83,9 @@
     "ZOOM_IN_OUT": "Zoom in/out",
     "PREVIOUS": "Precedente (←)",
     "NEXT": "Successivo (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "Ente Photos",
+    "TITLE_ALBUMS": "Ente Photos",
+    "TITLE_AUTH": "Ente Auth",
     "UPLOAD_FIRST_PHOTO": "Carica la tua prima foto",
     "IMPORT_YOUR_FOLDERS": "Importa una cartella",
     "UPLOAD_DROPZONE_MESSAGE": "Rilascia per eseguire il backup dei file",

apps/auth/public/locales/nl/translation.json

@@ -83,9 +83,9 @@
     "ZOOM_IN_OUT": "In/uitzoomen",
     "PREVIOUS": "Vorige (←)",
     "NEXT": "Volgende (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "Ente Photos",
+    "TITLE_ALBUMS": "Ente Photos",
+    "TITLE_AUTH": "Ente Auth",
     "UPLOAD_FIRST_PHOTO": "Je eerste foto uploaden",
     "IMPORT_YOUR_FOLDERS": "Importeer uw mappen",
     "UPLOAD_DROPZONE_MESSAGE": "Sleep om een back-up van je bestanden te maken",

apps/auth/src/pages/auth/index.tsx

@@ -2,7 +2,7 @@ import React, { useContext, useEffect, useState } from 'react';
 import OTPDisplay from 'components/OTPDisplay';
 import { getAuthCodes } from 'services';
 import { CustomError } from '@ente/shared/error';
-import { PHOTOS_PAGES as PAGES } from '@ente/shared/constants/pages';
+import { AUTH_PAGES as PAGES } from '@ente/shared/constants/pages';
 import { useRouter } from 'next/router';
 import { AuthFooter } from 'components/AuthFooter';
 import { AppContext } from 'pages/_app';

apps/photos/package.json

@@ -27,7 +27,7 @@
         "bs58": "^5.0.0",
         "chrono-node": "^2.2.6",
         "comlink": "^4.3.0",
-        "debounce-promise": "^3.1.2",
+        "debounce": "^2.0.0",
         "density-clustering": "^1.3.0",
         "eventemitter3": "^4.0.7",
         "exifr": "^7.1.3",
@@ -51,6 +51,7 @@
         "ml-matrix": "^6.10.4",
         "next-transpile-modules": "^10.0.0",
         "otpauth": "^9.0.2",
+        "p-debounce": "^4.0.0",
         "p-queue": "^7.1.0",
         "photoswipe": "file:./thirdparty/photoswipe",
         "piexifjs": "^1.0.6",
@@ -81,7 +82,6 @@
     "devDependencies": {
         "@next/bundle-analyzer": "^13.4.12",
         "@types/bs58": "^4.0.1",
-        "@types/debounce-promise": "^3.1.3",
         "@types/leaflet": "^1.9.3",
         "@types/libsodium-wrappers": "^0.7.8",
         "@types/node": "^14.6.4",

apps/photos/public/locales/de/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "Generierung von Verschlüsselungsschlüsseln...",
     "PASSPHRASE_HINT": "Passwort",
     "CONFIRM_PASSPHRASE": "Passwort bestätigen",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "Die Passwörter stimmen nicht überein",
     "CONSOLE_WARNING_STOP": "STOPP!",
     "CONSOLE_WARNING_DESC": "",
@@ -83,9 +85,9 @@
     "ZOOM_IN_OUT": "Herein-/Herauszoomen",
     "PREVIOUS": "",
     "NEXT": "",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "",
+    "TITLE_ALBUMS": "",
+    "TITLE_AUTH": "",
     "UPLOAD_FIRST_PHOTO": "Lade dein erstes Foto hoch",
     "IMPORT_YOUR_FOLDERS": "Importiere deiner Ordner",
     "UPLOAD_DROPZONE_MESSAGE": "",
@@ -423,7 +425,6 @@
     "FILES": "Dateien",
     "EACH": "",
     "DEDUPLICATE_BASED_ON_SIZE": "",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "",
     "STOP_ALL_UPLOADS_MESSAGE": "",
     "STOP_UPLOADS_HEADER": "Hochladen stoppen?",
     "YES_STOP_UPLOADS": "Ja, Hochladen stoppen",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/en/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "Generating encryption keys...",
     "PASSPHRASE_HINT": "Password",
     "CONFIRM_PASSPHRASE": "Confirm password",
+    "REFERRAL_CODE_HINT": "How did you hear about Ente? (optional)",
+    "REFERRAL_INFO": "We don't track app installs, It'd help us if you told us where you found us!",
     "PASSPHRASE_MATCH_ERROR": "Passwords don't match",
     "CONSOLE_WARNING_STOP": "STOP!",
     "CONSOLE_WARNING_DESC": "This is a browser feature intended for developers. Please don't copy-paste unverified code here.",
@@ -83,9 +85,9 @@
     "ZOOM_IN_OUT": "Zoom in/out",
     "PREVIOUS": "Previous (←)",
     "NEXT": "Next (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "Ente Photos",
+    "TITLE_ALBUMS": "Ente Photos",
+    "TITLE_AUTH": "Ente Auth",
     "UPLOAD_FIRST_PHOTO": "Upload your first photo",
     "IMPORT_YOUR_FOLDERS": "Import your folders",
     "UPLOAD_DROPZONE_MESSAGE": "Drop to backup your files",
@@ -223,10 +225,10 @@
     "SELECTED": "selected",
     "VIDEO_PLAYBACK_FAILED_DOWNLOAD_INSTEAD": "This video cannot be played on your browser",
     "PEOPLE": "People",
-    "INDEXING_SCHEDULED": "indexing is scheduled...",
-    "ANALYZING_PHOTOS": "analyzing new photos {{indexStatus.nSyncedFiles}} of {{indexStatus.nTotalFiles}} done)...",
-    "INDEXING_PEOPLE": "indexing people in {{indexStatus.nSyncedFiles}} photos...",
-    "INDEXING_DONE": "indexed {{indexStatus.nSyncedFiles}} photos",
+    "INDEXING_SCHEDULED": "Indexing is scheduled...",
+    "ANALYZING_PHOTOS": "Indexing photos ({{indexStatus.nSyncedFiles,number}} / {{indexStatus.nTotalFiles,number}})",
+    "INDEXING_PEOPLE": "Indexing people in {{indexStatus.nSyncedFiles,number}} photos...",
+    "INDEXING_DONE": "Indexed {{indexStatus.nSyncedFiles,number}} photos",
     "UNIDENTIFIED_FACES": "unidentified faces",
     "OBJECTS": "objects",
     "TEXT": "text",
@@ -423,7 +425,6 @@
     "FILES": "Files",
     "EACH": "Each",
     "DEDUPLICATE_BASED_ON_SIZE": "The following files were clubbed based on their sizes, please review and delete items you believe are duplicates",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "The following files were clubbed based on their sizes and capture time, please review and delete items you believe are duplicates",
     "STOP_ALL_UPLOADS_MESSAGE": "Are you sure that you want to stop all the uploads in progress?",
     "STOP_UPLOADS_HEADER": "Stop uploads?",
     "YES_STOP_UPLOADS": "Yes, stop uploads",
@@ -540,7 +541,7 @@
     "COLLECT_PHOTOS": "Collect photos",
     "PUBLIC_COLLECT_SUBTEXT": "Allow people with the link to also add photos to the shared album.",
     "STOP_EXPORT": "Stop",
-    "EXPORT_PROGRESS": "<a>{{progress.success}} / {{progress.total}}</a> items synced",
+    "EXPORT_PROGRESS": "<a>{{progress.success, number}} / {{progress.total, number}}</a> items synced",
     "MIGRATING_EXPORT": "Preparing...",
     "RENAMING_COLLECTION_FOLDERS": "Renaming album folders...",
     "TRASHING_DELETED_FILES": "Trashing deleted files...",
@@ -619,8 +620,8 @@
     "ROTATION": "Rotation",
     "RESET": "Reset",
     "PHOTO_EDITOR": "Photo Editor",
-    "FASTER_UPLOAD":"Faster uploads",
-    "FASTER_UPLOAD_DESCRIPTION":"Route uploads through nearby servers",
+    "FASTER_UPLOAD": "Faster uploads",
+    "FASTER_UPLOAD_DESCRIPTION": "Route uploads through nearby servers",
     "STATUS": "Status",
     "INDEXED_ITEMS": "Indexed items",
     "CAST_ALBUM_TO_TV": "Play album on TV",
@@ -633,5 +634,6 @@
     "CHOOSE_DEVICE_FROM_BROWSER": "Choose a cast-compatible device from the browser popup.",
     "PAIR_WITH_PIN_WORKS_FOR_ANY_LARGE_SCREEN_DEVICE": "Pair with PIN works for any large screen device you want to play your album on.",
     "VISIT_CAST_ENTE_IO": "Visit cast.ente.io on the device you want to pair.",
-    "CAST_AUTO_PAIR_FAILED": "Chromecast Auto Pair failed. Please try again."
+    "CAST_AUTO_PAIR_FAILED": "Chromecast Auto Pair failed. Please try again.",
+    "CACHE_DIRECTORY": "Cache folder"
 }

apps/photos/public/locales/es/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "Generando claves de encriptación...",
     "PASSPHRASE_HINT": "Contraseña",
     "CONFIRM_PASSPHRASE": "Confirmar contraseña",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "Las contraseñas no coinciden",
     "CONSOLE_WARNING_STOP": "STOP!",
     "CONSOLE_WARNING_DESC": "Esta es una característica del navegador destinada a los desarrolladores. Por favor, no copie y pegue código sin verificar aquí.",
@@ -382,11 +384,11 @@
     "ADDED_AS": "",
     "COLLABORATOR_RIGHTS": "",
     "REMOVE_PARTICIPANT_HEAD": "",
-    "OWNER": "",
-    "COLLABORATORS": "",
-    "ADD_MORE": "",
+    "OWNER": "Propietario",
+    "COLLABORATORS": "Colaboradores",
+    "ADD_MORE": "Añadir más",
     "VIEWERS": "",
-    "OR_ADD_EXISTING": "",
+    "OR_ADD_EXISTING": "O elige uno existente",
     "REMOVE_PARTICIPANT_MESSAGE": "",
     "NOT_FOUND": "404 - No Encontrado",
     "LINK_EXPIRED": "Enlace expirado",
@@ -397,7 +399,7 @@
     "LINK_PASSWORD_LOCK": "Contraseña bloqueada",
     "PUBLIC_COLLECT": "Permitir añadir fotos",
     "LINK_DEVICE_LIMIT": "Límites del dispositivo",
-    "NO_DEVICE_LIMIT": "",
+    "NO_DEVICE_LIMIT": "Ninguno",
     "LINK_EXPIRY": "Enlace vencio",
     "NEVER": "Nunca",
     "DISABLE_FILE_DOWNLOAD": "Deshabilitar descarga",
@@ -406,7 +408,7 @@
     "COPYRIGHT": "Infracciones sobre los derechos de autor de alguien que estoy autorizado a representar",
     "SHARED_USING": "Compartido usando ",
     "ENTE_IO": "ente.io",
-    "SHARING_REFERRAL_CODE": "",
+    "SHARING_REFERRAL_CODE": "Usa el código <strong>{{referralCode}}</strong> para obtener 10 GB gratis",
     "LIVE": "VIVO",
     "DISABLE_PASSWORD": "Desactivar contraseña",
     "DISABLE_PASSWORD_MESSAGE": "Seguro que quieres cambiar la contrasena?",
@@ -423,13 +425,12 @@
     "FILES": "Archivos",
     "EACH": "Cada",
     "DEDUPLICATE_BASED_ON_SIZE": "Los siguientes archivos fueron organizados en base a sus tamaños, por favor revise y elimine elementos que cree que son duplicados",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "Los siguientes archivos fueron organizados en base a sus tamaños y tiempo de captura, por favor revise y elimine elementos que cree que son duplicados",
     "STOP_ALL_UPLOADS_MESSAGE": "¿Está seguro que desea detener todas las subidas en curso?",
     "STOP_UPLOADS_HEADER": "Detener las subidas?",
     "YES_STOP_UPLOADS": "Sí, detener las subidas",
-    "STOP_DOWNLOADS_HEADER": "",
-    "YES_STOP_DOWNLOADS": "",
-    "STOP_ALL_DOWNLOADS_MESSAGE": "",
+    "STOP_DOWNLOADS_HEADER": "¿Detener las descargas?",
+    "YES_STOP_DOWNLOADS": "Sí, detener las descargas",
+    "STOP_ALL_DOWNLOADS_MESSAGE": "¿Estás seguro de que quieres detener todas las descargas en curso?",
     "albums_one": "1 álbum",
     "albums_other": "{{count}} álbumes",
     "ALL_ALBUMS": "Todos los álbumes",
@@ -574,9 +575,9 @@
     "AUTH_NEXT": "siguiente",
     "AUTH_DOWNLOAD_MOBILE_APP": "Descarga nuestra aplicación móvil para administrar tus secretos",
     "HIDDEN": "",
-    "HIDE": "",
-    "UNHIDE": "",
-    "UNHIDE_TO_COLLECTION": "",
+    "HIDE": "Ocultar",
+    "UNHIDE": "Mostrar",
+    "UNHIDE_TO_COLLECTION": "Hacer visible al álbum",
     "SORT_BY": "",
     "NEWEST_FIRST": "",
     "OLDEST_FIRST": "",
@@ -594,7 +595,7 @@
     "NEW_YEAR": "",
     "NEW_YEAR_EVE": "",
     "IMAGE": "",
-    "VIDEO": "",
+    "VIDEO": "Video",
     "LIVE_PHOTO": "",
     "CONVERT": "",
     "CONFIRM_EDITOR_CLOSE_MESSAGE": "",
@@ -613,8 +614,8 @@
     "DOWNLOAD_EDITED": "",
     "SAVE_A_COPY_TO_ENTE": "",
     "RESTORE_ORIGINAL": "",
-    "TRANSFORM": "",
-    "COLORS": "",
+    "TRANSFORM": "Transformar",
+    "COLORS": "Colores",
     "FLIP": "",
     "ROTATION": "",
     "RESET": "",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/fa/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "",
     "PASSPHRASE_HINT": "",
     "CONFIRM_PASSPHRASE": "",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "",
     "CONSOLE_WARNING_STOP": "",
     "CONSOLE_WARNING_DESC": "",
@@ -423,7 +425,6 @@
     "FILES": "",
     "EACH": "",
     "DEDUPLICATE_BASED_ON_SIZE": "",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "",
     "STOP_ALL_UPLOADS_MESSAGE": "",
     "STOP_UPLOADS_HEADER": "",
     "YES_STOP_UPLOADS": "",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/fi/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "",
     "PASSPHRASE_HINT": "",
     "CONFIRM_PASSPHRASE": "",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "",
     "CONSOLE_WARNING_STOP": "",
     "CONSOLE_WARNING_DESC": "",
@@ -423,7 +425,6 @@
     "FILES": "",
     "EACH": "",
     "DEDUPLICATE_BASED_ON_SIZE": "",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "",
     "STOP_ALL_UPLOADS_MESSAGE": "",
     "STOP_UPLOADS_HEADER": "",
     "YES_STOP_UPLOADS": "",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/fr/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "Génération des clés de chiffrement...",
     "PASSPHRASE_HINT": "Mot de passe",
     "CONFIRM_PASSPHRASE": "Confirmer le mot de passe",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "Les mots de passe ne correspondent pas",
     "CONSOLE_WARNING_STOP": "STOP!",
     "CONSOLE_WARNING_DESC": "Ceci est une fonction de navigateur dédiée aux développeurs. Veuillez ne pas copier-coller un code non vérifié à cet endroit.",
@@ -83,9 +85,9 @@
     "ZOOM_IN_OUT": "Zoom +/-",
     "PREVIOUS": "Précédent (←)",
     "NEXT": "Suivant (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "",
+    "TITLE_ALBUMS": "",
+    "TITLE_AUTH": "",
     "UPLOAD_FIRST_PHOTO": "Chargez votre 1ere photo",
     "IMPORT_YOUR_FOLDERS": "Importez vos dossiers",
     "UPLOAD_DROPZONE_MESSAGE": "Déposez pour sauvegarder vos fichiers",
@@ -423,7 +425,6 @@
     "FILES": "Fichiers",
     "EACH": "Chacun",
     "DEDUPLICATE_BASED_ON_SIZE": "Les fichiers suivants ont été clubbed, basé sur leurs tailles, veuillez corriger et supprimer les objets que vous pensez être dupliqués",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "Les fichiers suivants ont été clubbed, basé sur leurs tailles et de l'heure de capture, veuillez corriger et supprimer les objets que vous pensez être dupliqués",
     "STOP_ALL_UPLOADS_MESSAGE": "Êtes-vous certains de vouloir arrêter tous les chargements en cours?",
     "STOP_UPLOADS_HEADER": "Arrêter les chargements ?",
     "YES_STOP_UPLOADS": "Oui, arrêter tout",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "Chargements plus rapides",
     "FASTER_UPLOAD_DESCRIPTION": "Router les chargements vers les serveurs à proximité",
     "STATUS": "État",
-    "INDEXED_ITEMS": "Éléments indexés"
+    "INDEXED_ITEMS": "Éléments indexés",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/it/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "Generazione delle chiavi di crittografia...",
     "PASSPHRASE_HINT": "Password",
     "CONFIRM_PASSPHRASE": "Conferma la password",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "Le password non corrispondono",
     "CONSOLE_WARNING_STOP": "STOP!",
     "CONSOLE_WARNING_DESC": "Questa è una funzionalità del browser destinata agli sviluppatori. Non copiare né incollare codice non verificato qui.",
@@ -83,9 +85,9 @@
     "ZOOM_IN_OUT": "Zoom in/out",
     "PREVIOUS": "Precedente (←)",
     "NEXT": "Successivo (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "",
+    "TITLE_ALBUMS": "",
+    "TITLE_AUTH": "",
     "UPLOAD_FIRST_PHOTO": "Carica la tua prima foto",
     "IMPORT_YOUR_FOLDERS": "Importa una cartella",
     "UPLOAD_DROPZONE_MESSAGE": "Rilascia per eseguire il backup dei file",
@@ -423,7 +425,6 @@
     "FILES": "",
     "EACH": "",
     "DEDUPLICATE_BASED_ON_SIZE": "",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "",
     "STOP_ALL_UPLOADS_MESSAGE": "",
     "STOP_UPLOADS_HEADER": "",
     "YES_STOP_UPLOADS": "",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/nl/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "Encryptiecodes worden gegenereerd...",
     "PASSPHRASE_HINT": "Wachtwoord",
     "CONFIRM_PASSPHRASE": "Wachtwoord bevestigen",
+    "REFERRAL_CODE_HINT": "Hoe hoorde je over Ente? (optioneel)",
+    "REFERRAL_INFO": "Wij gebruiken geen tracking. Het zou helpen als je ons vertelt waar je ons gevonden hebt!",
     "PASSPHRASE_MATCH_ERROR": "Wachtwoorden komen niet overeen",
     "CONSOLE_WARNING_STOP": "STOP!",
     "CONSOLE_WARNING_DESC": "Dit is een browserfunctie bedoeld voor ontwikkelaars. Gelieve hier geen niet-geverifieerde code te kopiëren/plakken.",
@@ -83,9 +85,9 @@
     "ZOOM_IN_OUT": "In/uitzoomen",
     "PREVIOUS": "Vorige (←)",
     "NEXT": "Volgende (→)",
-    "TITLE_PHOTOS": "ente Photos",
-    "TITLE_ALBUMS": "ente Photos",
-    "TITLE_AUTH": "ente Auth",
+    "TITLE_PHOTOS": "",
+    "TITLE_ALBUMS": "",
+    "TITLE_AUTH": "",
     "UPLOAD_FIRST_PHOTO": "Je eerste foto uploaden",
     "IMPORT_YOUR_FOLDERS": "Importeer uw mappen",
     "UPLOAD_DROPZONE_MESSAGE": "Sleep om een back-up van je bestanden te maken",
@@ -157,7 +159,7 @@
     "RENEWAL_ACTIVE_SUBSCRIPTION_STATUS": "Vernieuwt op {{date, dateTime}}",
     "RENEWAL_CANCELLED_SUBSCRIPTION_STATUS": "Eindigt op {{date, dateTime}}",
     "RENEWAL_CANCELLED_SUBSCRIPTION_INFO": "Uw abonnement loopt af op {{date, dateTime}}",
-    "ADD_ON_AVAILABLE_TILL": "",
+    "ADD_ON_AVAILABLE_TILL": "Jouw {{storage, string}} add-on is geldig tot {{date, dateTime}}",
     "STORAGE_QUOTA_EXCEEDED_SUBSCRIPTION_INFO": "U heeft uw opslaglimiet overschreden, gelieve <a>upgraden</a>",
     "SUBSCRIPTION_PURCHASE_SUCCESS": "<p>We hebben uw betaling ontvangen</p><p>Uw abonnement is geldig tot <strong>{{date, dateTime}}</strong></p>",
     "SUBSCRIPTION_PURCHASE_CANCELLED": "Uw aankoop is geannuleerd, probeer het opnieuw als u zich wilt abonneren",
@@ -172,7 +174,7 @@
     "UPDATE_SUBSCRIPTION": "Abonnement wijzigen",
     "CANCEL_SUBSCRIPTION": "Abonnement opzeggen",
     "CANCEL_SUBSCRIPTION_MESSAGE": "<p>Al je gegevens zullen worden verwijderd van onze servers aan het einde van deze factureringsperiode.</p><p>Weet u zeker dat u uw abonnement wilt opzeggen?</p>",
-    "CANCEL_SUBSCRIPTION_WITH_ADDON_MESSAGE": "",
+    "CANCEL_SUBSCRIPTION_WITH_ADDON_MESSAGE": "<p>Weet je zeker dat je je abonnement wilt opzeggen?</p>",
     "SUBSCRIPTION_CANCEL_FAILED": "Abonnement opzeggen mislukt",
     "SUBSCRIPTION_CANCEL_SUCCESS": "Abonnement succesvol geannuleerd",
     "REACTIVATE_SUBSCRIPTION": "Abonnement opnieuw activeren",
@@ -423,7 +425,6 @@
     "FILES": "Bestanden",
     "EACH": "Elke",
     "DEDUPLICATE_BASED_ON_SIZE": "De volgende bestanden zijn samengevoegd op basis van hun groottes. Controleer en verwijder items waarvan je denkt dat ze dubbel zijn",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "De volgende bestanden zijn samengevoegd op basis van hun groottes en opnametijd, bekijk en verwijder items waarvan je denkt dat ze dubbel zijn",
     "STOP_ALL_UPLOADS_MESSAGE": "Weet u zeker dat u wilt stoppen met alle uploads die worden uitgevoerd?",
     "STOP_UPLOADS_HEADER": "Stoppen met uploaden?",
     "YES_STOP_UPLOADS": "Ja, stop uploaden",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "Snellere uploads",
     "FASTER_UPLOAD_DESCRIPTION": "Uploaden door nabije servers",
     "STATUS": "Status",
-    "INDEXED_ITEMS": "Geïndexeerde bestanden"
+    "INDEXED_ITEMS": "Geïndexeerde bestanden",
+    "CACHE_DIRECTORY": "Cache map"
 }

apps/photos/public/locales/pt/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "",
     "PASSPHRASE_HINT": "",
     "CONFIRM_PASSPHRASE": "",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "",
     "CONSOLE_WARNING_STOP": "PARAR!",
     "CONSOLE_WARNING_DESC": "",
@@ -423,7 +425,6 @@
     "FILES": "",
     "EACH": "",
     "DEDUPLICATE_BASED_ON_SIZE": "",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "",
     "STOP_ALL_UPLOADS_MESSAGE": "",
     "STOP_UPLOADS_HEADER": "",
     "YES_STOP_UPLOADS": "",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/ru/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "",
     "PASSPHRASE_HINT": "",
     "CONFIRM_PASSPHRASE": "",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "",
     "CONSOLE_WARNING_STOP": "",
     "CONSOLE_WARNING_DESC": "",
@@ -423,7 +425,6 @@
     "FILES": "",
     "EACH": "",
     "DEDUPLICATE_BASED_ON_SIZE": "",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "",
     "STOP_ALL_UPLOADS_MESSAGE": "",
     "STOP_UPLOADS_HEADER": "",
     "YES_STOP_UPLOADS": "",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/tr/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "",
     "PASSPHRASE_HINT": "",
     "CONFIRM_PASSPHRASE": "",
+    "REFERRAL_CODE_HINT": "",
+    "REFERRAL_INFO": "",
     "PASSPHRASE_MATCH_ERROR": "",
     "CONSOLE_WARNING_STOP": "",
     "CONSOLE_WARNING_DESC": "",
@@ -423,7 +425,6 @@
     "FILES": "",
     "EACH": "",
     "DEDUPLICATE_BASED_ON_SIZE": "",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "",
     "STOP_ALL_UPLOADS_MESSAGE": "",
     "STOP_UPLOADS_HEADER": "",
     "YES_STOP_UPLOADS": "",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "",
     "FASTER_UPLOAD_DESCRIPTION": "",
     "STATUS": "",
-    "INDEXED_ITEMS": ""
+    "INDEXED_ITEMS": "",
+    "CACHE_DIRECTORY": ""
 }

apps/photos/public/locales/zh/translation.json

@@ -38,6 +38,8 @@
     "KEY_GENERATION_IN_PROGRESS_MESSAGE": "正在生成加密密钥...",
     "PASSPHRASE_HINT": "密码",
     "CONFIRM_PASSPHRASE": "请确认密码",
+    "REFERRAL_CODE_HINT": "您是如何知道Ente的？ （可选的）",
+    "REFERRAL_INFO": "我们不跟踪应用程序安装情况，如果您告诉我们您是在哪里找到我们的，将会有所帮助！",
     "PASSPHRASE_MATCH_ERROR": "两次输入的密码不一致",
     "CONSOLE_WARNING_STOP": "停止！",
     "CONSOLE_WARNING_DESC": "这是专为开发人员设计的浏览器功能。 请不要在此处复制粘贴未经验证的代码。",
@@ -423,7 +425,6 @@
     "FILES": "文件",
     "EACH": "每个",
     "DEDUPLICATE_BASED_ON_SIZE": "以下文件根据大小进行了合并，请检查并删除您认为重复的项目",
-    "DEDUPLICATE_BASED_ON_SIZE_AND_CAPTURE_TIME": "以下文件是根据它们的大小和捕获时间合并的，请检查并删除您认为重复的项目",
     "STOP_ALL_UPLOADS_MESSAGE": "您确定要停止所有正在进行的上传吗？",
     "STOP_UPLOADS_HEADER": "要停止上传吗？",
     "YES_STOP_UPLOADS": "是的，停止上传",
@@ -622,5 +623,6 @@
     "FASTER_UPLOAD": "更快上传",
     "FASTER_UPLOAD_DESCRIPTION": "通过附近的服务器路由上传",
     "STATUS": "状态",
-    "INDEXED_ITEMS": "索引项目"
+    "INDEXED_ITEMS": "索引项目",
+    "CACHE_DIRECTORY": "缓存文件夹"
 }

apps/photos/public/manifest.json

@@ -1,6 +1,6 @@
 {
-    "short_name": "ente Photos",
-    "name": "ente Photos | encrypted photo storage",
+    "short_name": "Ente Photos",
+    "name": "Ente Photos | Safe home for your Photos",
     "icons": [
         {
             "src": "/images/ente/192.png",

apps/photos/public/offline.html

@@ -1,7 +1,7 @@
 <!DOCTYPE html>
 <html>
     <head>
-        <title>ente Photos</title>
+        <title>Ente Photos</title>
         <meta name="viewport" content="width=device-width, initial-scale=1" />
         <style>
             * {

apps/photos/src/components/Collections/CollectionCard.tsx

@@ -1,11 +1,8 @@
-import React from 'react';
-import { GalleryContext } from 'pages/gallery';
-import { useState, useContext, useEffect } from 'react';
-import downloadManager from 'services/downloadManager';
+import { useState, useEffect } from 'react';
+import downloadManager from 'services/download';
 import { EnteFile } from 'types/file';
 import { StaticThumbnail } from 'components/PlaceholderThumbnails';
 import { LoadingThumbnail } from 'components/PlaceholderThumbnails';
-import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery';
 
 export default function CollectionCard(props: {
     children?: any;
@@ -23,28 +20,19 @@ export default function CollectionCard(props: {
     } = props;
 
     const [coverImageURL, setCoverImageURL] = useState(null);
-    const galleryContext = useContext(GalleryContext);
-    const publicCollectionGalleryContext = useContext(
-        PublicCollectionGalleryContext
-    );
-
-    const thumbsStore = publicCollectionGalleryContext?.accessedThroughSharedURL
-        ? publicCollectionGalleryContext.thumbs
-        : galleryContext.thumbs;
 
     useEffect(() => {
         const main = async () => {
             if (!file) {
                 return;
             }
-            if (!thumbsStore.has(file.id)) {
-                if (isScrolling) {
-                    return;
-                }
-                const url = await downloadManager.getThumbnail(file);
-                thumbsStore.set(file.id, url);
+            const url = await downloadManager.getThumbnailForPreview(
+                file,
+                isScrolling
+            );
+            if (url) {
+                setCoverImageURL(url);
             }
-            setCoverImageURL(thumbsStore.get(file.id));
         };
         main();
     }, [file, isScrolling]);

apps/photos/src/components/Directory/changeOption.tsx

@@ -0,0 +1,26 @@
+import OverflowMenu from '@ente/shared/components/OverflowMenu/menu';
+import { OverflowMenuOption } from '@ente/shared/components/OverflowMenu/option';
+import MoreHoriz from '@mui/icons-material/MoreHoriz';
+import { t } from 'i18next';
+import FolderIcon from '@mui/icons-material/Folder';
+
+export default function ChangeDirectoryOption({
+    changeExportDirectory: changeDirectory,
+}) {
+    return (
+        <OverflowMenu
+            triggerButtonProps={{
+                sx: {
+                    ml: 1,
+                },
+            }}
+            ariaControls={'export-option'}
+            triggerButtonIcon={<MoreHoriz />}>
+            <OverflowMenuOption
+                onClick={changeDirectory}
+                startIcon={<FolderIcon />}>
+                {t('CHANGE_FOLDER')}
+            </OverflowMenuOption>
+        </OverflowMenu>
+    );
+}

apps/photos/src/components/Directory/index.tsx

@@ -0,0 +1,34 @@
+import { styled } from '@mui/material/styles';
+import LinkButton from '@ente/shared/components/LinkButton';
+import { Tooltip } from '@mui/material';
+import ElectronAPIs from '@ente/shared/electron';
+import { logError } from '@ente/shared/sentry';
+
+const DirectoryPathContainer = styled(LinkButton)(
+    ({ width }) => `
+    width: ${width}px;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    /* Beginning of string */
+    direction: rtl;
+    text-align: left;
+`
+);
+
+export const DirectoryPath = ({ width, path }) => {
+    const handleClick = async () => {
+        try {
+            await ElectronAPIs.openDirectory(path);
+        } catch (e) {
+            logError(e, 'openDirectory failed');
+        }
+    };
+    return (
+        <DirectoryPathContainer width={width} onClick={handleClick}>
+            <Tooltip title={path}>
+                <span>{path}</span>
+            </Tooltip>
+        </DirectoryPathContainer>
+    );
+};

apps/photos/src/components/DropdownInput.tsx

@@ -20,7 +20,7 @@ interface Iprops<T> {
     options: DropdownOption<T>[];
     message?: string;
     messageProps?: TypographyProps;
-    selected: string;
+    selected: T;
     setSelected: (selectedValue: T) => void;
     placeholder?: string;
 }

apps/photos/src/components/ExportModal.tsx

@@ -1,5 +1,5 @@
 import isElectron from 'is-electron';
-import React, { useEffect, useState, useContext } from 'react';
+import { useEffect, useState, useContext } from 'react';
 import exportService from 'services/export';
 import { ExportProgress, ExportSettings } from 'types/export';
 import {
@@ -8,9 +8,7 @@ import {
     Dialog,
     DialogContent,
     Divider,
-    styled,
     Switch,
-    Tooltip,
     Typography,
 } from '@mui/material';
 import { logError } from '@ente/shared/sentry';
@@ -21,29 +19,16 @@ import {
 import ExportFinished from './ExportFinished';
 import ExportInit from './ExportInit';
 import ExportInProgress from './ExportInProgress';
-import FolderIcon from '@mui/icons-material/Folder';
 import { ExportStage } from 'constants/export';
 import DialogTitleWithCloseButton from '@ente/shared/components/DialogBox/TitleWithCloseButton';
-import MoreHoriz from '@mui/icons-material/MoreHoriz';
-import OverflowMenu from '@ente/shared/components/OverflowMenu/menu';
-import { OverflowMenuOption } from '@ente/shared/components/OverflowMenu/option';
 import { AppContext } from 'pages/_app';
 import { getExportDirectoryDoesNotExistMessage } from 'utils/ui';
 import { t } from 'i18next';
-import LinkButton from './pages/gallery/LinkButton';
 import { CustomError } from '@ente/shared/error';
 import { addLogLine } from '@ente/shared/logging';
 import { EnteFile } from 'types/file';
-
-const ExportFolderPathContainer = styled(LinkButton)`
-    width: 262px;
-    white-space: nowrap;
-    overflow: hidden;
-    text-overflow: ellipsis;
-    /* Beginning of string */
-    direction: rtl;
-    text-align: left;
-`;
+import ChangeDirectoryOption from './Directory/changeOption';
+import { DirectoryPath } from './Directory';
 
 interface Props {
     show: boolean;
@@ -163,10 +148,6 @@ export default function ExportModal(props: Props) {
         }
     };
 
-    const handleOpenExportDirectoryClick = () => {
-        void exportService.openExportDirectory(exportFolder);
-    };
-
     const toggleContinuousExport = () => {
         try {
             verifyExportFolderExists();
@@ -207,7 +188,6 @@ export default function ExportModal(props: Props) {
                     exportFolder={exportFolder}
                     changeExportDirectory={handleChangeExportDirectoryClick}
                     exportStage={exportStage}
-                    openExportDirectory={handleOpenExportDirectoryClick}
                 />
                 <ContinuousExport
                     continuousExport={continuousExport}
@@ -229,12 +209,7 @@ export default function ExportModal(props: Props) {
     );
 }
 
-function ExportDirectory({
-    exportFolder,
-    changeExportDirectory,
-    exportStage,
-    openExportDirectory,
-}) {
+function ExportDirectory({ exportFolder, changeExportDirectory, exportStage }) {
     return (
         <SpaceBetweenFlex minHeight={'48px'}>
             <Typography color="text.muted" mr={'16px'}>
@@ -247,16 +222,10 @@ function ExportDirectory({
                     </Button>
                 ) : (
                     <VerticallyCenteredFlex>
-                        <ExportFolderPathContainer
-                            onClick={openExportDirectory}>
-                            <Tooltip title={exportFolder}>
-                                <span>{exportFolder}</span>
-                            </Tooltip>
-                        </ExportFolderPathContainer>
-
+                        <DirectoryPath width={262} path={exportFolder} />
                         {exportStage === ExportStage.FINISHED ||
                         exportStage === ExportStage.INIT ? (
-                            <ExportDirectoryOption
+                            <ChangeDirectoryOption
                                 changeExportDirectory={changeExportDirectory}
                             />
                         ) : (
@@ -269,25 +238,6 @@ function ExportDirectory({
     );
 }
 
-function ExportDirectoryOption({ changeExportDirectory }) {
-    return (
-        <OverflowMenu
-            triggerButtonProps={{
-                sx: {
-                    ml: 1,
-                },
-            }}
-            ariaControls={'export-option'}
-            triggerButtonIcon={<MoreHoriz />}>
-            <OverflowMenuOption
-                onClick={changeExportDirectory}
-                startIcon={<FolderIcon />}>
-                {t('CHANGE_FOLDER')}
-            </OverflowMenuOption>
-        </OverflowMenu>
-    );
-}
-
 function ContinuousExport({ continuousExport, toggleContinuousExport }) {
     return (
         <SpaceBetweenFlex minHeight={'48px'}>

apps/photos/src/components/MachineLearning/ImageViews.tsx

@@ -3,7 +3,11 @@ import { Skeleton, styled } from '@mui/material';
 
 import { imageBitmapToBlob } from 'utils/image';
 import { logError } from '@ente/shared/sentry';
-import { getBlobFromCache } from '@ente/shared/storage/cacheStorage/helpers';
+import { cached } from '@ente/shared/storage/cacheStorage/helpers';
+import machineLearningService from 'services/machineLearning/machineLearningService';
+import { LS_KEYS, getData } from '@ente/shared/storage/localStorage';
+import { User } from '@ente/shared/user/types';
+import { addLogLine } from '@ente/shared/logging';
 
 export const FaceCropsRow = styled('div')`
     & > img {
@@ -19,19 +23,44 @@ export const FaceImagesRow = styled('div')`
     }
 `;
 
-export function ImageCacheView(props: { url: string; cacheName: string }) {
+export function ImageCacheView(props: {
+    url: string;
+    cacheName: string;
+    faceID: string;
+}) {
     const [imageBlob, setImageBlob] = useState<Blob>();
 
     useEffect(() => {
         let didCancel = false;
-
         async function loadImage() {
             try {
+                const user: User = getData(LS_KEYS.USER);
                 let blob: Blob;
-                if (!props.url || !props.cacheName) {
+                if (!props.url || !props.cacheName || !user) {
                     blob = undefined;
                 } else {
-                    blob = await getBlobFromCache(props.cacheName, props.url);
+                    blob = await cached(
+                        props.cacheName,
+                        props.url,
+                        async () => {
+                            try {
+                                addLogLine(
+                                    'ImageCacheView: regenerate face crop',
+                                    props.faceID
+                                );
+                                return machineLearningService.regenerateFaceCrop(
+                                    user.token,
+                                    user.id,
+                                    props.faceID
+                                );
+                            } catch (e) {
+                                logError(
+                                    e,
+                                    'ImageCacheView: regenerate face crop failed'
+                                );
+                            }
+                        }
+                    );
                 }
 
                 !didCancel && setImageBlob(blob);

apps/photos/src/components/MachineLearning/PeopleList.tsx

@@ -65,6 +65,7 @@ export const PeopleList = React.memo((props: PeopleListProps) => {
                     <ImageCacheView
                         url={person.displayImageUrl}
                         cacheName={CACHES.FACE_CROPS}
+                        faceID={person.displayFaceId}
                     />
                 </FaceChip>
             ))}
@@ -172,6 +173,7 @@ export function UnidentifiedFaces(props: {
                     faces.map((face, index) => (
                         <FaceChip key={index}>
                             <ImageCacheView
+                                faceID={face.id}
                                 url={face.crop?.imageUrl}
                                 cacheName={CACHES.FACE_CROPS}
                             />

apps/photos/src/components/Menu/EnteMenuItem.tsx

@@ -12,11 +12,18 @@ import {
     VerticallyCenteredFlex,
 } from '@ente/shared/components/Container';
 import React from 'react';
+import ChangeDirectoryOption from 'components/Directory/changeOption';
 
 interface Iprops {
     onClick: () => void;
     color?: ButtonProps['color'];
-    variant?: 'primary' | 'captioned' | 'toggle' | 'secondary' | 'mini';
+    variant?:
+        | 'primary'
+        | 'captioned'
+        | 'toggle'
+        | 'secondary'
+        | 'mini'
+        | 'path';
     fontWeight?: TypographyProps['fontWeight'];
     startIcon?: React.ReactNode;
     endIcon?: React.ReactNode;
@@ -41,14 +48,24 @@ export function EnteMenuItem({
     labelComponent,
     disabled = false,
 }: Iprops) {
-    const handleClick = () => {
+    const handleButtonClick = () => {
+        if (variant === 'path' || variant === 'toggle') {
+            return;
+        }
+        onClick();
+    };
+
+    const handleIconClick = () => {
+        if (variant !== 'path' && variant !== 'toggle') {
+            return;
+        }
         onClick();
     };
 
     return (
         <MenuItem
             disabled={disabled}
-            onClick={handleClick}
+            onClick={handleButtonClick}
             sx={{
                 width: '100%',
                 color: (theme) =>
@@ -93,7 +110,15 @@ export function EnteMenuItem({
                 <VerticallyCenteredFlex gap={'4px'}>
                     {endIcon && endIcon}
                     {variant === 'toggle' && (
-                        <PublicShareSwitch checked={checked} />
+                        <PublicShareSwitch
+                            checked={checked}
+                            onClick={handleIconClick}
+                        />
+                    )}
+                    {variant === 'path' && (
+                        <ChangeDirectoryOption
+                            changeExportDirectory={handleIconClick}
+                        />
                     )}
                 </VerticallyCenteredFlex>
             </SpaceBetweenFlex>

apps/photos/src/components/PhotoFrame.tsx

@@ -3,22 +3,27 @@ import PreviewCard from './pages/gallery/PreviewCard';
 import { useContext, useEffect, useState } from 'react';
 import { EnteFile } from 'types/file';
 import { styled } from '@mui/material';
-import DownloadManager from 'services/downloadManager';
+import DownloadManager, {
+    LivePhotoSourceURL,
+    SourceURLs,
+} from 'services/download';
 import AutoSizer from 'react-virtualized-auto-sizer';
 import PhotoViewer from 'components/PhotoViewer';
 import { TRASH_SECTION } from 'constants/collection';
 import { updateFileMsrcProps, updateFileSrcProps } from 'utils/photoFrame';
-import { PhotoList } from './PhotoList';
-import { MergedSourceURL, SelectedState } from 'types/gallery';
-import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
+import { SelectedState } from 'types/gallery';
 import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery';
 import { useRouter } from 'next/router';
 import { logError } from '@ente/shared/sentry';
 import { addLogLine } from '@ente/shared/logging';
 import PhotoSwipe from 'photoswipe';
 import useMemoSingleThreaded from '@ente/shared/hooks/useMemoSingleThreaded';
-import { getPlayableVideo } from 'utils/file';
 import { FILE_TYPE } from 'constants/file';
+import { PHOTOS_PAGES } from '@ente/shared/constants/pages';
+import { PhotoList } from './PhotoList';
+import { DedupePhotoList } from './PhotoList/dedupe';
+import { Duplicate } from 'services/deduplicationService';
+import { CustomError } from '@ente/shared/error';
 
 const Container = styled('div')`
     display: block;
@@ -36,7 +41,12 @@ const Container = styled('div')`
 const PHOTOSWIPE_HASH_SUFFIX = '&opened';
 
 interface Props {
+    page:
+        | PHOTOS_PAGES.GALLERY
+        | PHOTOS_PAGES.DEDUPLICATE
+        | PHOTOS_PAGES.SHARED_ALBUMS;
     files: EnteFile[];
+    duplicates?: Duplicate[];
     syncWithRemote: () => Promise<void>;
     favItemIds?: Set<number>;
     setSelected: (
@@ -55,6 +65,8 @@ interface Props {
 }
 
 const PhotoFrame = ({
+    page,
+    duplicates,
     files,
     syncWithRemote,
     favItemIds,
@@ -73,6 +85,9 @@ const PhotoFrame = ({
     const [open, setOpen] = useState(false);
     const [currentIndex, setCurrentIndex] = useState<number>(0);
     const [fetching, setFetching] = useState<{ [k: number]: boolean }>({});
+    const [thumbFetching, setThumbFetching] = useState<{
+        [k: number]: boolean;
+    }>({});
     const galleryContext = useContext(GalleryContext);
     const publicCollectionGalleryContext = useContext(
         PublicCollectionGalleryContext
@@ -82,14 +97,6 @@ const PhotoFrame = ({
     const [isShiftKeyPressed, setIsShiftKeyPressed] = useState(false);
     const router = useRouter();
 
-    const thumbsStore = publicCollectionGalleryContext?.accessedThroughSharedURL
-        ? publicCollectionGalleryContext.thumbs
-        : galleryContext.thumbs;
-
-    const filesStore = publicCollectionGalleryContext?.accessedThroughSharedURL
-        ? publicCollectionGalleryContext.files
-        : galleryContext.files;
-
     const displayFiles = useMemoSingleThreaded(() => {
         return files.map((item) => {
             const filteredItem = {
@@ -98,22 +105,13 @@ const PhotoFrame = ({
                 h: window.innerHeight,
                 title: item.pubMagicMetadata?.data.caption,
             };
-            try {
-                if (thumbsStore.has(item.id)) {
-                    updateFileMsrcProps(filteredItem, thumbsStore.get(item.id));
-                }
-                if (filesStore.has(item.id)) {
-                    updateFileSrcProps(filteredItem, filesStore.get(item.id));
-                }
-            } catch (e) {
-                logError(e, 'PhotoFrame url prefill failed');
-            }
             return filteredItem;
         });
     }, [files]);
 
     useEffect(() => {
         setFetching({});
+        setThumbFetching({});
     }, [displayFiles]);
 
     useEffect(() => {
@@ -182,21 +180,12 @@ const PhotoFrame = ({
             // this is to prevent outdated updateURL call from updating the wrong file
             if (file.id !== id) {
                 addLogLine(
-                    `PhotoSwipe: updateURL: file id mismatch: ${file.id} !== ${id}`
+                    `[${id}]PhotoSwipe: updateURL: file id mismatch: ${file.id} !== ${id}`
                 );
-                return;
+                throw Error(CustomError.UPDATE_URL_FILE_ID_MISMATCH);
             }
-            if (file.msrc && file.msrc !== url && !forceUpdate) {
-                addLogLine(
-                    `PhotoSwipe: updateURL: msrc already set: ${file.msrc}`
-                );
-                logError(
-                    new Error(
-                        `PhotoSwipe: updateURL: msrc already set: ${file.msrc}`
-                    ),
-                    'PhotoSwipe: updateURL called with msrc already set'
-                );
-                return;
+            if (file.msrc && !forceUpdate) {
+                throw Error(CustomError.URL_ALREADY_SET);
             }
             updateFileMsrcProps(file, url);
         };
@@ -204,42 +193,25 @@ const PhotoFrame = ({
     const updateSrcURL = async (
         index: number,
         id: number,
-        mergedSrcURL: MergedSourceURL,
+        srcURLs: SourceURLs,
         forceUpdate?: boolean
     ) => {
         const file = displayFiles[index];
         // this is to prevent outdate updateSrcURL call from updating the wrong file
         if (file.id !== id) {
             addLogLine(
-                `PhotoSwipe: updateSrcURL: file id mismatch: ${file.id} !== ${id}`
+                `[${id}]PhotoSwipe: updateSrcURL: file id mismatch: ${file.id}`
             );
-            return;
+            throw Error(CustomError.UPDATE_URL_FILE_ID_MISMATCH);
         }
         if (file.isSourceLoaded && !forceUpdate) {
-            addLogLine(
-                `PhotoSwipe: updateSrcURL: source already loaded: ${file.id}`
-            );
-            logError(
-                new Error(
-                    `PhotoSwipe: updateSrcURL: source already loaded: ${file.id}`
-                ),
-                'PhotoSwipe updateSrcURL called when source already loaded'
-            );
-            return;
+            throw Error(CustomError.URL_ALREADY_SET);
         } else if (file.conversionFailed) {
-            addLogLine(
-                `PhotoSwipe: updateSrcURL: conversion failed: ${file.id}`
-            );
-            logError(
-                new Error(
-                    `PhotoSwipe: updateSrcURL: conversion failed: ${file.id}`
-                ),
-                'PhotoSwipe updateSrcURL called when conversion failed'
-            );
-            return;
+            addLogLine(`[${id}]PhotoSwipe: updateSrcURL: conversion failed`);
+            throw Error(CustomError.FILE_CONVERSION_FAILED);
         }
 
-        await updateFileSrcProps(file, mergedSrcURL);
+        await updateFileSrcProps(file, srcURLs);
     };
 
     const handleClose = (needUpdate) => {
@@ -380,35 +352,20 @@ const PhotoFrame = ({
                 item.isSourceLoaded
             } fetching:${fetching[item.id]}`
         );
+
         if (!item.msrc) {
-            addLogLine(`[${item.id}] doesn't have thumbnail`);
             try {
-                let url: string;
-                if (thumbsStore.has(item.id)) {
+                if (thumbFetching[item.id]) {
                     addLogLine(
-                        `[${item.id}] gallery context cache hit, using cached thumb`
+                        `[${item.id}] thumb download already in progress`
                     );
-                    url = thumbsStore.get(item.id);
-                } else {
-                    addLogLine(
-                        `[${item.id}] gallery context cache miss, calling downloadManager to get thumb`
-                    );
-                    if (
-                        publicCollectionGalleryContext.accessedThroughSharedURL
-                    ) {
-                        url =
-                            await PublicCollectionDownloadManager.getThumbnail(
-                                item,
-                                publicCollectionGalleryContext.token,
-                                publicCollectionGalleryContext.passwordToken
-                            );
-                    } else {
-                        url = await DownloadManager.getThumbnail(item);
-                    }
-                    thumbsStore.set(item.id, url);
+                    return;
                 }
-                updateURL(index)(item.id, url);
+                addLogLine(`[${item.id}] doesn't have thumbnail`);
+                thumbFetching[item.id] = true;
+                const url = await DownloadManager.getThumbnailForPreview(item);
                 try {
+                    updateURL(index)(item.id, url);
                     addLogLine(
                         `[${
                             item.id
@@ -419,14 +376,17 @@ const PhotoFrame = ({
                         instance.updateSize(true);
                     }
                 } catch (e) {
-                    logError(
-                        e,
-                        'updating photoswipe after msrc url update failed'
-                    );
+                    if (e.message !== CustomError.URL_ALREADY_SET) {
+                        logError(
+                            e,
+                            'updating photoswipe after msrc url update failed'
+                        );
+                    }
                     // ignore
                 }
             } catch (e) {
                 logError(e, 'getSlideData failed get msrc url failed');
+                thumbFetching[item.id] = false;
             }
         }
 
@@ -447,51 +407,86 @@ const PhotoFrame = ({
         try {
             addLogLine(`[${item.id}] new file src request`);
             fetching[item.id] = true;
-            let srcURL: MergedSourceURL;
-            if (filesStore.has(item.id)) {
-                addLogLine(
-                    `[${item.id}] gallery context cache hit, using cached file`
-                );
-                srcURL = filesStore.get(item.id);
-            } else {
-                addLogLine(
-                    `[${item.id}] gallery context cache miss, calling downloadManager to get file`
-                );
-                let downloadedURL: {
-                    original: string[];
-                    converted: string[];
+            const srcURLs = await DownloadManager.getFileForPreview(item);
+            if (item.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
+                const srcImgURL = srcURLs.url as LivePhotoSourceURL;
+                const imageURL = await srcImgURL.image();
+
+                const dummyImgSrcUrl: SourceURLs = {
+                    url: imageURL,
+                    isOriginal: false,
+                    isRenderable: !!imageURL,
+                    type: 'normal',
                 };
-                if (publicCollectionGalleryContext.accessedThroughSharedURL) {
-                    downloadedURL =
-                        await PublicCollectionDownloadManager.getFile(
-                            item,
-                            publicCollectionGalleryContext.token,
-                            publicCollectionGalleryContext.passwordToken,
-                            true
+                try {
+                    await updateSrcURL(index, item.id, dummyImgSrcUrl);
+                    addLogLine(
+                        `[${item.id}] calling invalidateCurrItems for live photo imgSrc, source loaded :${item.isSourceLoaded}`
+                    );
+                    instance.invalidateCurrItems();
+                    if ((instance as any).isOpen()) {
+                        instance.updateSize(true);
+                    }
+                } catch (e) {
+                    if (e.message !== CustomError.URL_ALREADY_SET) {
+                        logError(
+                            e,
+                            'updating photoswipe after for live photo imgSrc update failed'
                         );
-                } else {
-                    downloadedURL = await DownloadManager.getFile(item, true);
+                    }
+                }
+                if (!imageURL) {
+                    // no image url, no need to load video
+                    return;
                 }
-                const mergedURL: MergedSourceURL = {
-                    original: downloadedURL.original.join(','),
-                    converted: downloadedURL.converted.join(','),
-                };
-                filesStore.set(item.id, mergedURL);
-                srcURL = mergedURL;
-            }
-            await updateSrcURL(index, item.id, srcURL);
 
-            try {
-                addLogLine(
-                    `[${item.id}] calling invalidateCurrItems for src, source loaded :${item.isSourceLoaded}`
-                );
-                instance.invalidateCurrItems();
-                if ((instance as any).isOpen()) {
-                    instance.updateSize(true);
+                const videoURL = await srcImgURL.video();
+                const loadedLivePhotoSrcURL: SourceURLs = {
+                    url: { video: videoURL, image: imageURL },
+                    isOriginal: false,
+                    isRenderable: !!videoURL,
+                    type: 'livePhoto',
+                };
+                try {
+                    await updateSrcURL(
+                        index,
+                        item.id,
+                        loadedLivePhotoSrcURL,
+                        true
+                    );
+                    addLogLine(
+                        `[${item.id}] calling invalidateCurrItems for live photo complete, source loaded :${item.isSourceLoaded}`
+                    );
+                    instance.invalidateCurrItems();
+                    if ((instance as any).isOpen()) {
+                        instance.updateSize(true);
+                    }
+                } catch (e) {
+                    if (e.message !== CustomError.URL_ALREADY_SET) {
+                        logError(
+                            e,
+                            'updating photoswipe for live photo complete update failed'
+                        );
+                    }
+                }
+            } else {
+                try {
+                    await updateSrcURL(index, item.id, srcURLs);
+                    addLogLine(
+                        `[${item.id}] calling invalidateCurrItems for src, source loaded :${item.isSourceLoaded}`
+                    );
+                    instance.invalidateCurrItems();
+                    if ((instance as any).isOpen()) {
+                        instance.updateSize(true);
+                    }
+                } catch (e) {
+                    if (e.message !== CustomError.URL_ALREADY_SET) {
+                        logError(
+                            e,
+                            'updating photoswipe after src url update failed'
+                        );
+                    }
                 }
-            } catch (e) {
-                logError(e, 'updating photoswipe after src url update failed');
-                throw e;
             }
         } catch (e) {
             logError(e, 'getSlideData failed get src url failed');
@@ -522,8 +517,8 @@ const PhotoFrame = ({
             );
             return;
         }
-        updateURL(index)(item.id, item.msrc, true);
         try {
+            updateURL(index)(item.id, item.msrc, true);
             addLogLine(
                 `[${
                     item.id
@@ -534,7 +529,9 @@ const PhotoFrame = ({
                 instance.updateSize(true);
             }
         } catch (e) {
-            logError(e, 'updating photoswipe after msrc url update failed');
+            if (e.message !== CustomError.URL_ALREADY_SET) {
+                logError(e, 'updating photoswipe after msrc url update failed');
+            }
             // ignore
         }
         try {
@@ -542,48 +539,11 @@ const PhotoFrame = ({
                 `[${item.id}] new file getConvertedVideo request- ${item.metadata.title}}`
             );
             fetching[item.id] = true;
-            if (!filesStore.has(item.id)) {
-                addLogLine(
-                    `[${item.id}] getConvertedVideo called for file that is not downloaded`
-                );
-                logError(
-                    new Error(),
-                    'getConvertedVideo called for file that is not downloaded'
-                );
-                // this should never happen, convert video button should not be visible if file is not downloaded
-                return;
-            }
 
-            const srcURL = filesStore.get(item.id);
-            let originalVideoURL;
-            if (item.metadata.fileType === FILE_TYPE.VIDEO) {
-                originalVideoURL = srcURL.original;
-            } else {
-                originalVideoURL = srcURL.original.split(',')[1];
-            }
-            const playableVideo = await getPlayableVideo(
-                item.metadata.title,
-                await (await fetch(originalVideoURL)).blob(),
-                true
-            );
-            const convertedVideoURL = playableVideo
-                ? URL.createObjectURL(playableVideo)
-                : '';
-            if (item.metadata.fileType === FILE_TYPE.VIDEO) {
-                srcURL.converted = convertedVideoURL;
-            } else {
-                const prvConvertedImageURL = srcURL.converted.split(',')[0];
-                srcURL.converted = [
-                    prvConvertedImageURL,
-                    convertedVideoURL,
-                ].join(',');
-            }
-
-            filesStore.set(item.id, srcURL);
-
-            await updateSrcURL(index, item.id, srcURL, true);
+            const srcURL = await DownloadManager.getFileForPreview(item, true);
 
             try {
+                await updateSrcURL(index, item.id, srcURL, true);
                 addLogLine(
                     `[${item.id}] calling invalidateCurrItems for src, source loaded :${item.isSourceLoaded}`
                 );
@@ -592,7 +552,12 @@ const PhotoFrame = ({
                     instance.updateSize(true);
                 }
             } catch (e) {
-                logError(e, 'updating photoswipe after src url update failed');
+                if (e.message !== CustomError.URL_ALREADY_SET) {
+                    logError(
+                        e,
+                        'updating photoswipe after src url update failed'
+                    );
+                }
                 throw e;
             }
         } catch (e) {
@@ -605,16 +570,27 @@ const PhotoFrame = ({
     return (
         <Container>
             <AutoSizer>
-                {({ height, width }) => (
-                    <PhotoList
-                        width={width}
-                        height={height}
-                        getThumbnail={getThumbnail}
-                        displayFiles={displayFiles}
-                        activeCollectionID={activeCollectionID}
-                        showAppDownloadBanner={showAppDownloadBanner}
-                    />
-                )}
+                {({ height, width }) =>
+                    page === PHOTOS_PAGES.DEDUPLICATE ? (
+                        <DedupePhotoList
+                            width={width}
+                            height={height}
+                            getThumbnail={getThumbnail}
+                            duplicates={duplicates}
+                            activeCollectionID={activeCollectionID}
+                            showAppDownloadBanner={showAppDownloadBanner}
+                        />
+                    ) : (
+                        <PhotoList
+                            width={width}
+                            height={height}
+                            getThumbnail={getThumbnail}
+                            displayFiles={displayFiles}
+                            activeCollectionID={activeCollectionID}
+                            showAppDownloadBanner={showAppDownloadBanner}
+                        />
+                    )
+                }
             </AutoSizer>
             <PhotoViewer
                 isOpen={open}

apps/photos/src/components/PhotoList/dedupe.tsx

@@ -0,0 +1,365 @@
+import React, { useRef, useEffect, useState, useMemo } from 'react';
+import {
+    VariableSizeList as List,
+    ListChildComponentProps,
+    areEqual,
+} from 'react-window';
+import { Box, styled } from '@mui/material';
+import { EnteFile } from 'types/file';
+import {
+    IMAGE_CONTAINER_MAX_HEIGHT,
+    MIN_COLUMNS,
+    DATE_CONTAINER_HEIGHT,
+    GAP_BTW_TILES,
+    SPACE_BTW_DATES,
+    SIZE_AND_COUNT_CONTAINER_HEIGHT,
+    IMAGE_CONTAINER_MAX_WIDTH,
+} from 'constants/gallery';
+import { convertBytesToHumanReadable } from '@ente/shared/utils/size';
+import { FlexWrapper } from '@ente/shared/components/Container';
+import { t } from 'i18next';
+import memoize from 'memoize-one';
+import { Duplicate } from 'services/deduplicationService';
+
+export enum ITEM_TYPE {
+    TIME = 'TIME',
+    FILE = 'FILE',
+    SIZE_AND_COUNT = 'SIZE_AND_COUNT',
+    HEADER = 'HEADER',
+    FOOTER = 'FOOTER',
+    MARKETING_FOOTER = 'MARKETING_FOOTER',
+    OTHER = 'OTHER',
+}
+
+export interface TimeStampListItem {
+    itemType: ITEM_TYPE;
+    items?: EnteFile[];
+    itemStartIndex?: number;
+    date?: string;
+    dates?: {
+        date: string;
+        span: number;
+    }[];
+    groups?: number[];
+    item?: any;
+    id?: string;
+    height?: number;
+    fileSize?: number;
+    fileCount?: number;
+}
+
+const ListItem = styled('div')`
+    display: flex;
+    justify-content: center;
+`;
+
+const getTemplateColumns = (
+    columns: number,
+    shrinkRatio: number,
+    groups?: number[]
+): string => {
+    if (groups) {
+        // need to confirm why this was there
+        // const sum = groups.reduce((acc, item) => acc + item, 0);
+        // if (sum < columns) {
+        //     groups[groups.length - 1] += columns - sum;
+        // }
+        return groups
+            .map(
+                (x) =>
+                    `repeat(${x}, ${IMAGE_CONTAINER_MAX_WIDTH * shrinkRatio}px)`
+            )
+            .join(` ${SPACE_BTW_DATES}px `);
+    } else {
+        return `repeat(${columns},${
+            IMAGE_CONTAINER_MAX_WIDTH * shrinkRatio
+        }px)`;
+    }
+};
+
+function getFractionFittableColumns(width: number): number {
+    return (
+        (width - 2 * getGapFromScreenEdge(width) + GAP_BTW_TILES) /
+        (IMAGE_CONTAINER_MAX_WIDTH + GAP_BTW_TILES)
+    );
+}
+
+function getGapFromScreenEdge(width: number) {
+    if (width > MIN_COLUMNS * IMAGE_CONTAINER_MAX_WIDTH) {
+        return 24;
+    } else {
+        return 4;
+    }
+}
+
+function getShrinkRatio(width: number, columns: number) {
+    return (
+        (width -
+            2 * getGapFromScreenEdge(width) -
+            (columns - 1) * GAP_BTW_TILES) /
+        (columns * IMAGE_CONTAINER_MAX_WIDTH)
+    );
+}
+
+const ListContainer = styled(Box)<{
+    columns: number;
+    shrinkRatio: number;
+    groups?: number[];
+}>`
+    display: grid;
+    grid-template-columns: ${({ columns, shrinkRatio, groups }) =>
+        getTemplateColumns(columns, shrinkRatio, groups)};
+    grid-column-gap: ${GAP_BTW_TILES}px;
+    width: 100%;
+    color: #fff;
+    padding: 0 24px;
+    @media (max-width: ${IMAGE_CONTAINER_MAX_WIDTH * MIN_COLUMNS}px) {
+        padding: 0 4px;
+    }
+`;
+
+const ListItemContainer = styled(FlexWrapper)<{ span: number }>`
+    grid-column: span ${(props) => props.span};
+`;
+
+const DateContainer = styled(ListItemContainer)`
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+    height: ${DATE_CONTAINER_HEIGHT}px;
+    color: ${({ theme }) => theme.colors.text.muted};
+`;
+
+const SizeAndCountContainer = styled(DateContainer)`
+    margin-top: 1rem;
+    height: ${SIZE_AND_COUNT_CONTAINER_HEIGHT}px;
+`;
+
+interface Props {
+    height: number;
+    width: number;
+    duplicates: Duplicate[];
+    showAppDownloadBanner: boolean;
+    getThumbnail: (
+        file: EnteFile,
+        index: number,
+        isScrolling?: boolean
+    ) => JSX.Element;
+    activeCollectionID: number;
+}
+
+interface ItemData {
+    timeStampList: TimeStampListItem[];
+    columns: number;
+    shrinkRatio: number;
+    renderListItem: (
+        timeStampListItem: TimeStampListItem,
+        isScrolling?: boolean
+    ) => JSX.Element;
+}
+
+const createItemData = memoize(
+    (
+        timeStampList: TimeStampListItem[],
+        columns: number,
+        shrinkRatio: number,
+        renderListItem: (
+            timeStampListItem: TimeStampListItem,
+            isScrolling?: boolean
+        ) => JSX.Element
+    ): ItemData => ({
+        timeStampList,
+        columns,
+        shrinkRatio,
+        renderListItem,
+    })
+);
+const PhotoListRow = React.memo(
+    ({
+        index,
+        style,
+        isScrolling,
+        data,
+    }: ListChildComponentProps<ItemData>) => {
+        const { timeStampList, columns, shrinkRatio, renderListItem } = data;
+        return (
+            <ListItem style={style}>
+                <ListContainer
+                    columns={columns}
+                    shrinkRatio={shrinkRatio}
+                    groups={timeStampList[index].groups}>
+                    {renderListItem(timeStampList[index], isScrolling)}
+                </ListContainer>
+            </ListItem>
+        );
+    },
+    areEqual
+);
+
+const getTimeStampListFromDuplicates = (duplicates: Duplicate[], columns) => {
+    const timeStampList: TimeStampListItem[] = [];
+    for (let index = 0; index < duplicates.length; index++) {
+        const dupes = duplicates[index];
+        timeStampList.push({
+            itemType: ITEM_TYPE.SIZE_AND_COUNT,
+            fileSize: dupes.size,
+            fileCount: dupes.files.length,
+        });
+        let lastIndex = 0;
+        while (lastIndex < dupes.files.length) {
+            timeStampList.push({
+                itemType: ITEM_TYPE.FILE,
+                items: dupes.files.slice(lastIndex, lastIndex + columns),
+                itemStartIndex: index,
+            });
+            lastIndex += columns;
+        }
+    }
+    return timeStampList;
+};
+
+export function DedupePhotoList({
+    height,
+    width,
+    duplicates,
+    getThumbnail,
+    activeCollectionID,
+}: Props) {
+    const [timeStampList, setTimeStampList] = useState<TimeStampListItem[]>([]);
+    const refreshInProgress = useRef(false);
+    const shouldRefresh = useRef(false);
+    const listRef = useRef(null);
+
+    const columns = useMemo(() => {
+        const fittableColumns = getFractionFittableColumns(width);
+        let columns = Math.floor(fittableColumns);
+        if (columns < MIN_COLUMNS) {
+            columns = MIN_COLUMNS;
+        }
+        return columns;
+    }, [width]);
+
+    const shrinkRatio = getShrinkRatio(width, columns);
+    const listItemHeight =
+        IMAGE_CONTAINER_MAX_HEIGHT * shrinkRatio + GAP_BTW_TILES;
+
+    const refreshList = () => {
+        listRef.current?.resetAfterIndex(0);
+    };
+
+    useEffect(() => {
+        const main = () => {
+            if (refreshInProgress.current) {
+                shouldRefresh.current = true;
+                return;
+            }
+            refreshInProgress.current = true;
+            const timeStampList = getTimeStampListFromDuplicates(
+                duplicates,
+                columns
+            );
+            setTimeStampList(timeStampList);
+            refreshInProgress.current = false;
+            if (shouldRefresh.current) {
+                shouldRefresh.current = false;
+                setTimeout(main, 0);
+            }
+        };
+        main();
+    }, [columns, duplicates]);
+
+    useEffect(() => {
+        refreshList();
+    }, [timeStampList]);
+
+    const getItemSize = (timeStampList) => (index) => {
+        switch (timeStampList[index].itemType) {
+            case ITEM_TYPE.TIME:
+                return DATE_CONTAINER_HEIGHT;
+            case ITEM_TYPE.SIZE_AND_COUNT:
+                return SIZE_AND_COUNT_CONTAINER_HEIGHT;
+            case ITEM_TYPE.FILE:
+                return listItemHeight;
+            default:
+                return timeStampList[index].height;
+        }
+    };
+
+    const generateKey = (index) => {
+        switch (timeStampList[index].itemType) {
+            case ITEM_TYPE.FILE:
+                return `${timeStampList[index].items[0].id}-${
+                    timeStampList[index].items.slice(-1)[0].id
+                }`;
+            default:
+                return `${timeStampList[index].id}-${index}`;
+        }
+    };
+
+    const renderListItem = (
+        listItem: TimeStampListItem,
+        isScrolling: boolean
+    ) => {
+        switch (listItem.itemType) {
+            case ITEM_TYPE.SIZE_AND_COUNT:
+                return (
+                    <SizeAndCountContainer span={columns}>
+                        {listItem.fileCount} {t('FILES')},{' '}
+                        {convertBytesToHumanReadable(listItem.fileSize || 0)}{' '}
+                        {t('EACH')}
+                    </SizeAndCountContainer>
+                );
+            case ITEM_TYPE.FILE: {
+                const ret = listItem.items.map((item, idx) =>
+                    getThumbnail(
+                        item,
+                        listItem.itemStartIndex + idx,
+                        isScrolling
+                    )
+                );
+                if (listItem.groups) {
+                    let sum = 0;
+                    for (let i = 0; i < listItem.groups.length - 1; i++) {
+                        sum = sum + listItem.groups[i];
+                        ret.splice(
+                            sum,
+                            0,
+                            <div key={`${listItem.items[0].id}-gap-${i}`} />
+                        );
+                        sum += 1;
+                    }
+                }
+                return ret;
+            }
+            default:
+                return listItem.item;
+        }
+    };
+
+    if (!timeStampList?.length) {
+        return <></>;
+    }
+
+    const itemData = createItemData(
+        timeStampList,
+        columns,
+        shrinkRatio,
+        renderListItem
+    );
+
+    return (
+        <List
+            key={`${activeCollectionID}`}
+            itemData={itemData}
+            ref={listRef}
+            itemSize={getItemSize(timeStampList)}
+            height={height}
+            width={width}
+            itemCount={timeStampList.length}
+            itemKey={generateKey}
+            overscanCount={3}
+            useIsScrolling>
+            {PhotoListRow}
+        </List>
+    );
+}

apps/photos/src/components/PhotoList.tsx → apps/photos/src/components/PhotoList/index.tsx

@@ -19,14 +19,12 @@ import {
 import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery';
 import { ENTE_WEBSITE_LINK } from '@ente/shared/constants/urls';
 import { convertBytesToHumanReadable } from '@ente/shared/utils/size';
-import { DeduplicateContext } from 'pages/deduplicate';
 import { FlexWrapper } from '@ente/shared/components/Container';
 import { Typography } from '@mui/material';
 import { GalleryContext } from 'pages/gallery';
 import { formatDate } from '@ente/shared/time/format';
 import { Trans } from 'react-i18next';
 import { t } from 'i18next';
-import { areFilesWithFileHashSame, hasFileHash } from 'utils/upload';
 import memoize from 'memoize-one';
 
 const A_DAY = 24 * 60 * 60 * 1000;
@@ -261,7 +259,6 @@ export function PhotoList({
     const publicCollectionGalleryContext = useContext(
         PublicCollectionGalleryContext
     );
-    const deduplicateContext = useContext(DeduplicateContext);
 
     const [timeStampList, setTimeStampList] = useState<TimeStampListItem[]>([]);
     const refreshInProgress = useRef(false);
@@ -306,9 +303,6 @@ export function PhotoList({
             }
             if (galleryContext.isClipSearchResult) {
                 noGrouping(timeStampList);
-            } else if (deduplicateContext.isOnDeduplicatePage) {
-                skipMerge = true;
-                groupByFileSize(timeStampList);
             } else {
                 groupByTime(timeStampList);
             }
@@ -345,9 +339,6 @@ export function PhotoList({
         width,
         height,
         displayFiles,
-        deduplicateContext.isOnDeduplicatePage,
-        deduplicateContext.fileSizeMap,
-        deduplicateContext.clubSameTimeFilesOnly,
         galleryContext.photoListHeader,
         publicCollectionGalleryContext.photoListHeader,
         galleryContext.isClipSearchResult,
@@ -420,67 +411,6 @@ export function PhotoList({
         refreshList();
     }, [timeStampList]);
 
-    const groupByFileSize = (timeStampList: TimeStampListItem[]) => {
-        let index = 0;
-        while (index < displayFiles.length) {
-            const firstFile = displayFiles[index];
-            const firstFileSize = deduplicateContext.fileSizeMap.get(
-                firstFile.id
-            );
-            const firstFileCreationTime = firstFile.metadata.creationTime;
-            let lastFileIndex = index;
-
-            while (lastFileIndex < displayFiles.length) {
-                const lastFile = displayFiles[lastFileIndex];
-
-                const lastFileSize = deduplicateContext.fileSizeMap.get(
-                    lastFile.id
-                );
-                if (lastFileSize !== firstFileSize) {
-                    break;
-                }
-
-                const lastFileCreationTime = lastFile.metadata.creationTime;
-                if (
-                    deduplicateContext.clubSameTimeFilesOnly &&
-                    lastFileCreationTime !== firstFileCreationTime
-                ) {
-                    break;
-                }
-
-                const eitherFileHasFileHash =
-                    hasFileHash(lastFile.metadata) ||
-                    hasFileHash(firstFile.metadata);
-                if (
-                    eitherFileHasFileHash &&
-                    !areFilesWithFileHashSame(
-                        lastFile.metadata,
-                        firstFile.metadata
-                    )
-                ) {
-                    break;
-                }
-                lastFileIndex++;
-            }
-            lastFileIndex--;
-            timeStampList.push({
-                itemType: ITEM_TYPE.SIZE_AND_COUNT,
-                fileSize: firstFileSize,
-                fileCount: lastFileIndex - index + 1,
-            });
-
-            while (index <= lastFileIndex) {
-                const tileSize = Math.min(columns, lastFileIndex - index + 1);
-                timeStampList.push({
-                    itemType: ITEM_TYPE.FILE,
-                    items: displayFiles.slice(index, index + tileSize),
-                    itemStartIndex: index,
-                });
-                index += tileSize;
-            }
-        }
-    };
-
     const groupByTime = (timeStampList: TimeStampListItem[]) => {
         let listItemIndex = 0;
         let currentDate;

apps/photos/src/components/PhotoViewer/ImageEditorOverlay/index.tsx

@@ -19,7 +19,7 @@ import {
 } from 'react';
 
 import { EnteFile } from 'types/file';
-import downloadManager from 'services/downloadManager';
+import downloadManager from 'services/download';
 import { MenuItemGroup } from 'components/Menu/MenuItemGroup';
 import { EnteMenuItem } from 'components/Menu/EnteMenuItem';
 import CropOriginalIcon from '@mui/icons-material/CropOriginal';
@@ -210,12 +210,11 @@ const ImageEditorOverlay = (props: IProps) => {
             const ctx = canvasRef.current.getContext('2d');
             ctx.imageSmoothingEnabled = false;
             if (!fileURL) {
-                const { converted } = await downloadManager.getFile(
-                    props.file,
-                    true
+                const srcURLs = await downloadManager.getFileForPreview(
+                    props.file
                 );
-                img.src = converted[0];
-                setFileURL(converted[0]);
+                img.src = srcURLs.url as string;
+                setFileURL(srcURLs.url as string);
             } else {
                 img.src = fileURL;
             }

apps/photos/src/components/PhotoViewer/index.tsx

@@ -49,8 +49,7 @@ import { getParsedExifData } from 'services/upload/exifService';
 import { getFileType } from 'services/typeDetectionService';
 import { ConversionFailedNotification } from './styledComponents/ConversionFailedNotification';
 import { GalleryContext } from 'pages/gallery';
-import downloadManager from 'services/downloadManager';
-import publicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
+import downloadManager, { LoadedLivePhotoSourceURL } from 'services/download';
 import CircularProgressWithLabel from './styledComponents/CircularProgressWithLabel';
 import EnteSpinner from '@ente/shared/components/EnteSpinner';
 import AlbumOutlined from '@mui/icons-material/AlbumOutlined';
@@ -137,14 +136,10 @@ function PhotoViewer(props: Iprops) {
 
     const [showEditButton, setShowEditButton] = useState(false);
 
+    const [showZoomButton, setShowZoomButton] = useState(false);
+
     useEffect(() => {
-        if (publicCollectionGalleryContext.accessedThroughSharedURL) {
-            publicCollectionDownloadManager.setProgressUpdater(
-                setFileDownloadProgress
-            );
-        } else {
-            downloadManager.setProgressUpdater(setFileDownloadProgress);
-        }
+        downloadManager.setProgressUpdater(setFileDownloadProgress);
     }, []);
 
     useEffect(() => {
@@ -295,18 +290,26 @@ function PhotoViewer(props: Iprops) {
     }
 
     function updateExif(file: EnteFile) {
-        if (file.metadata.fileType !== FILE_TYPE.IMAGE) {
+        if (file.metadata.fileType === FILE_TYPE.VIDEO) {
             setExif({ key: file.src, value: null });
             return;
         }
-        if (
-            !file ||
-            !exifCopy?.current?.value === null ||
-            exifCopy?.current?.key === file.src
-        ) {
+        if (!file.isSourceLoaded || file.conversionFailed) {
+            return;
+        }
+
+        if (!file || !exifCopy?.current?.value === null) {
+            return;
+        }
+        const key =
+            file.metadata.fileType === FILE_TYPE.IMAGE
+                ? file.src
+                : (file.srcURLs.url as LoadedLivePhotoSourceURL).image;
+
+        if (exifCopy?.current?.key === key) {
             return;
         }
-        setExif({ key: file.src, value: undefined });
+        setExif({ key, value: undefined });
         checkExifAvailable(file);
     }
 
@@ -338,6 +341,10 @@ function PhotoViewer(props: Iprops) {
         );
     }
 
+    function updateShowZoomButton(file: EnteFile) {
+        setShowZoomButton(file.metadata.fileType === FILE_TYPE.IMAGE);
+    }
+
     const openPhotoSwipe = () => {
         const { items, currentIndex } = props;
         const options = {
@@ -411,6 +418,7 @@ function PhotoViewer(props: Iprops) {
             updateShowConvertBtn(currItem);
             updateIsSourceLoaded(currItem);
             updateShowEditButton(currItem);
+            updateShowZoomButton(currItem);
         });
         photoSwipe.listen('resize', () => {
             if (!photoSwipe?.currItem) return;
@@ -540,22 +548,28 @@ function PhotoViewer(props: Iprops) {
                 return;
             }
             try {
-                if (file.isSourceLoaded) {
-                    exifExtractionInProgress.current = file.src;
-                    const fileObject = await getFileFromURL(
-                        file.originalImageURL
+                exifExtractionInProgress.current = file.src;
+                let fileObject: File;
+                if (file.metadata.fileType === FILE_TYPE.IMAGE) {
+                    fileObject = await getFileFromURL(
+                        file.src as string,
+                        file.metadata.title
                     );
-                    const fileTypeInfo = await getFileType(fileObject);
-                    const exifData = await getParsedExifData(
-                        fileObject,
-                        fileTypeInfo
-                    );
-                    if (exifExtractionInProgress.current === file.src) {
-                        if (exifData) {
-                            setExif({ key: file.src, value: exifData });
-                        } else {
-                            setExif({ key: file.src, value: null });
-                        }
+                } else {
+                    const url = (file.srcURLs.url as LoadedLivePhotoSourceURL)
+                        .image;
+                    fileObject = await getFileFromURL(url, file.metadata.title);
+                }
+                const fileTypeInfo = await getFileType(fileObject);
+                const exifData = await getParsedExifData(
+                    fileObject,
+                    fileTypeInfo
+                );
+                if (exifExtractionInProgress.current === file.src) {
+                    if (exifData) {
+                        setExif({ key: file.src, value: exifData });
+                    } else {
+                        setExif({ key: file.src, value: null });
                     }
                 }
             } finally {
@@ -589,12 +603,7 @@ function PhotoViewer(props: Iprops) {
         if (file && props.enableDownload) {
             appContext.startLoading();
             try {
-                await downloadFile(
-                    file,
-                    publicCollectionGalleryContext.accessedThroughSharedURL,
-                    publicCollectionGalleryContext.token,
-                    publicCollectionGalleryContext.passwordToken
-                );
+                await downloadFile(file);
             } catch (e) {
                 // do nothing
             }
@@ -766,12 +775,14 @@ function PhotoViewer(props: Iprops) {
                                     <DeleteIcon />
                                 </button>
                             )}
-                            <button
-                                className="pswp__button pswp__button--custom"
-                                onClick={toggleZoomInAndOut}
-                                title={t('ZOOM_IN_OUT')}>
-                                <ZoomInOutlinedIcon />
-                            </button>
+                            {showZoomButton && (
+                                <button
+                                    className="pswp__button pswp__button--custom"
+                                    onClick={toggleZoomInAndOut}
+                                    title={t('ZOOM_IN_OUT')}>
+                                    <ZoomInOutlinedIcon />
+                                </button>
+                            )}
                             <button
                                 className="pswp__button pswp__button--custom"
                                 onClick={() => {

apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx

@@ -1,32 +1,27 @@
-import React, { useContext } from 'react';
+import { useContext } from 'react';
 import { PeopleList } from 'components/MachineLearning/PeopleList';
 import { IndexStatus } from 'types/machineLearning/ui';
 import { SuggestionType, Suggestion } from 'types/search';
 import { components } from 'react-select';
 import { Row } from '@ente/shared/components/Container';
-import { Col } from 'react-bootstrap';
 import { AppContext } from 'pages/_app';
 import styled from '@mui/styled-engine';
 import { t } from 'i18next';
+import { Box } from '@mui/material';
 
 const { Menu } = components;
 
-const LegendRow = styled(Row)`
-    align-items: center;
-    justify-content: space-between;
-    margin-bottom: 0px;
-`;
-
 const Legend = styled('span')`
     font-size: 20px;
     color: #ddd;
     display: inline;
+    padding: 0px 12px;
 `;
 
 const Caption = styled('span')`
     font-size: 12px;
     display: inline;
-    padding: 8px 12px;
+    padding: 0px 12px;
 `;
 
 const MenuWithPeople = (props) => {
@@ -44,17 +39,17 @@ const MenuWithPeople = (props) => {
     const indexStatus = indexStatusSuggestion?.value as IndexStatus;
     return (
         <Menu {...props}>
-            <Col>
+            <Box my={1}>
                 {((appContext.mlSearchEnabled && indexStatus) ||
                     (people && people.length > 0)) && (
-                    <LegendRow>
+                    <Box>
                         <Legend>{t('PEOPLE')}</Legend>
-                    </LegendRow>
+                    </Box>
                 )}
                 {appContext.mlSearchEnabled && indexStatus && (
-                    <LegendRow>
+                    <Box>
                         <Caption>{indexStatusSuggestion.label}</Caption>
-                    </LegendRow>
+                    </Box>
                 )}
                 {people && people.length > 0 && (
                     <Row>
@@ -68,7 +63,7 @@ const MenuWithPeople = (props) => {
                         />
                     </Row>
                 )}
-            </Col>
+            </Box>
             {props.children}
         </Menu>
     );

apps/photos/src/components/Search/SearchBar/searchInput/index.tsx

@@ -1,5 +1,5 @@
 import { IconButton } from '@mui/material';
-import debounce from 'debounce-promise';
+import pDebounce from 'p-debounce';
 import { AppContext } from 'pages/_app';
 import React, {
     useCallback,
@@ -83,7 +83,7 @@ export default function SearchInput(props: Iprops) {
         }
     };
 
-    const getOptions = debounce(
+    const getOptions = pDebounce(
         getAutoCompleteSuggestions(props.files, props.collections),
         250
     );

apps/photos/src/components/Sidebar/AdvancedSettings.tsx

@@ -17,6 +17,7 @@ import { ClipService } from 'services/clipService';
 import { VerticallyCenteredFlex } from '@ente/shared/components/Container';
 import { ClipExtractionStatus } from 'services/clipService';
 import { formatNumber } from 'utils/number/format';
+import CacheDirectory from './Preferences/CacheDirectory';
 
 export default function AdvancedSettings({ open, onClose, onRootClose }) {
     const appContext = useContext(AppContext);
@@ -78,19 +79,22 @@ export default function AdvancedSettings({ open, onClose, onRootClose }) {
                 <Box px={'8px'}>
                     <Stack py="20px" spacing="24px">
                         {isElectron() && (
-                            <Box>
-                                <MenuSectionTitle
-                                    title={t('LABS')}
-                                    icon={<ScienceIcon />}
-                                />
-                                <MenuItemGroup>
-                                    <EnteMenuItem
-                                        endIcon={<ChevronRight />}
-                                        onClick={openMlSearchSettings}
-                                        label={t('ML_SEARCH')}
+                            <>
+                                <CacheDirectory />
+                                <Box>
+                                    <MenuSectionTitle
+                                        title={t('LABS')}
+                                        icon={<ScienceIcon />}
                                     />
-                                </MenuItemGroup>
-                            </Box>
+                                    <MenuItemGroup>
+                                        <EnteMenuItem
+                                            endIcon={<ChevronRight />}
+                                            onClick={openMlSearchSettings}
+                                            label={t('ML_SEARCH')}
+                                        />
+                                    </MenuItemGroup>
+                                </Box>
+                            </>
                         )}
                         <Box>
                             <MenuItemGroup>
@@ -106,31 +110,37 @@ export default function AdvancedSettings({ open, onClose, onRootClose }) {
                             />
                         </Box>
 
-                        <Box>
-                            <MenuSectionTitle title={t('STATUS')} />
-                            <Stack py={'12px'} px={'12px'} spacing={'24px'}>
-                                <VerticallyCenteredFlex
-                                    justifyContent="space-between"
-                                    alignItems={'center'}>
-                                    <Typography>
-                                        {t('INDEXED_ITEMS')}
-                                    </Typography>
-                                    <Typography>
-                                        {formatNumber(indexingStatus.indexed)}
-                                    </Typography>
-                                </VerticallyCenteredFlex>
-                                <VerticallyCenteredFlex
-                                    justifyContent="space-between"
-                                    alignItems={'center'}>
-                                    <Typography>
-                                        {t('PENDING_ITEMS')}
-                                    </Typography>
-                                    <Typography>
-                                        {formatNumber(indexingStatus.pending)}
-                                    </Typography>
-                                </VerticallyCenteredFlex>
-                            </Stack>
-                        </Box>
+                        {isElectron() && (
+                            <Box>
+                                <MenuSectionTitle title={t('STATUS')} />
+                                <Stack py={'12px'} px={'12px'} spacing={'24px'}>
+                                    <VerticallyCenteredFlex
+                                        justifyContent="space-between"
+                                        alignItems={'center'}>
+                                        <Typography>
+                                            {t('INDEXED_ITEMS')}
+                                        </Typography>
+                                        <Typography>
+                                            {formatNumber(
+                                                indexingStatus.indexed
+                                            )}
+                                        </Typography>
+                                    </VerticallyCenteredFlex>
+                                    <VerticallyCenteredFlex
+                                        justifyContent="space-between"
+                                        alignItems={'center'}>
+                                        <Typography>
+                                            {t('PENDING_ITEMS')}
+                                        </Typography>
+                                        <Typography>
+                                            {formatNumber(
+                                                indexingStatus.pending
+                                            )}
+                                        </Typography>
+                                    </VerticallyCenteredFlex>
+                                </Stack>
+                            </Box>
+                        )}
                     </Stack>
                 </Box>
             </Stack>

apps/photos/src/components/Sidebar/Preferences/CacheDirectory.tsx

@@ -0,0 +1,60 @@
+import ElectronAPIs from '@ente/shared/electron';
+import { addLogLine } from '@ente/shared/logging';
+import { logError } from '@ente/shared/sentry';
+import Box from '@mui/material/Box';
+import { DirectoryPath } from 'components/Directory';
+import { EnteMenuItem } from 'components/Menu/EnteMenuItem';
+import { MenuItemGroup } from 'components/Menu/MenuItemGroup';
+import MenuSectionTitle from 'components/Menu/MenuSectionTitle';
+import { t } from 'i18next';
+import isElectron from 'is-electron';
+import { useEffect, useState } from 'react';
+import DownloadManager from 'services/download';
+
+export default function CacheDirectory() {
+    const [cacheDirectory, setCacheDirectory] = useState(undefined);
+
+    useEffect(() => {
+        const main = async () => {
+            if (isElectron()) {
+                const customCacheDirectory =
+                    await ElectronAPIs.getCacheDirectory();
+                setCacheDirectory(customCacheDirectory);
+            }
+        };
+        main();
+    }, []);
+
+    const handleCacheDirectoryChange = async () => {
+        try {
+            if (!isElectron()) {
+                return;
+            }
+            const newFolder = await ElectronAPIs.selectDirectory();
+            if (!newFolder) {
+                return;
+            }
+            addLogLine(`Export folder changed to ${newFolder}`);
+            await ElectronAPIs.setCustomCacheDirectory(newFolder);
+            setCacheDirectory(newFolder);
+            await DownloadManager.reloadCaches();
+        } catch (e) {
+            logError(e, 'handleCacheDirectoryChange failed');
+        }
+    };
+
+    return (
+        <Box>
+            <MenuSectionTitle title={t('CACHE_DIRECTORY')} />
+            <MenuItemGroup>
+                <EnteMenuItem
+                    variant="path"
+                    onClick={handleCacheDirectoryChange}
+                    labelComponent={
+                        <DirectoryPath width={265} path={cacheDirectory} />
+                    }
+                />
+            </MenuItemGroup>
+        </Box>
+    );
+}

apps/photos/src/components/Sidebar/Preferences/index.tsx

@@ -82,7 +82,8 @@ export default function Preferences({ open, onClose, onRootClose }) {
                             checked={!optOutOfCrashReports}
                             onClick={toggleOptOutOfCrashReports}
                             label={t('CRASH_REPORTING')}
-                        />{' '}
+                        />
+
                         <EnteMenuItem
                             onClick={openMapSettings}
                             endIcon={<ChevronRight />}

apps/photos/src/components/Sidebar/UtilitySection.tsx

@@ -64,8 +64,6 @@ export default function UtilitySection({ closeSidebar }) {
 
     const redirectToDeduplicatePage = () => router.push(PAGES.DEDUPLICATE);
 
-    const redirectToAuthenticatorPage = () => router.push(PAGES.AUTH);
-
     const somethingWentWrong = () =>
         setDialogMessage({
             title: t('ERROR'),
@@ -132,13 +130,6 @@ export default function UtilitySection({ closeSidebar }) {
                 label={t('DEDUPLICATE_FILES')}
             />
 
-            {isInternalUser() && (
-                <EnteMenuItem
-                    variant="secondary"
-                    onClick={redirectToAuthenticatorPage}
-                    label={t('AUTHENTICATOR_SECTION')}
-                />
-            )}
             <EnteMenuItem
                 variant="secondary"
                 onClick={openPreferencesOptions}

apps/photos/src/components/WatchFolder/index.tsx

@@ -12,6 +12,7 @@ import { UPLOAD_STRATEGY } from 'constants/upload';
 import { getImportSuggestion } from 'utils/upload';
 import ElectronAPIs from '@ente/shared/electron';
 import { PICKED_UPLOAD_TYPE } from 'constants/upload';
+import isElectron from 'is-electron';
 
 interface Iprops {
     open: boolean;
@@ -25,6 +26,9 @@ export default function WatchFolder({ open, onClose }: Iprops) {
     const appContext = useContext(AppContext);
 
     useEffect(() => {
+        if (!isElectron()) {
+            return;
+        }
         setMappings(watchFolderService.getWatchMappings());
     }, []);
 

apps/photos/src/components/pages/gallery/PlanSelector/card/free.tsx

@@ -7,12 +7,14 @@ import { PeriodToggler } from '../periodToggler';
 import Plans from '../plans';
 import { hasAddOnBonus } from 'utils/billing';
 import { BFAddOnRow } from '../plans/BfAddOnRow';
+import { ManageSubscription } from '../manageSubscription';
 
 export default function FreeSubscriptionPlanSelectorCard({
     plans,
     subscription,
     bonusData,
     closeModal,
+    setLoading,
     planPeriod,
     togglePeriod,
     onPlanSelect,
@@ -48,6 +50,14 @@ export default function FreeSubscriptionPlanSelectorCard({
                             closeModal={closeModal}
                         />
                     )}
+                    {hasAddOnBonus(bonusData) && (
+                        <ManageSubscription
+                            subscription={subscription}
+                            bonusData={bonusData}
+                            closeModal={closeModal}
+                            setLoading={setLoading}
+                        />
+                    )}
                 </Stack>
             </Box>
         </>

apps/photos/src/components/pages/gallery/PlanSelector/card/index.tsx

@@ -191,6 +191,7 @@ function PlanSelectorCard(props: Props) {
                         subscription={subscription}
                         bonusData={bonusData}
                         closeModal={props.closeModal}
+                        setLoading={props.setLoading}
                         planPeriod={planPeriod}
                         togglePeriod={togglePeriod}
                         onPlanSelect={onPlanSelect}

apps/photos/src/components/pages/gallery/PreviewCard.tsx

@@ -2,12 +2,10 @@ import React, { useContext, useEffect, useRef, useState } from 'react';
 import { EnteFile } from 'types/file';
 import { styled } from '@mui/material';
 import PlayCircleOutlineOutlinedIcon from '@mui/icons-material/PlayCircleOutlineOutlined';
-import DownloadManager from 'services/downloadManager';
+import DownloadManager from 'services/download';
 import useLongPress from '@ente/shared/hooks/useLongPress';
 import { GalleryContext } from 'pages/gallery';
 import { GAP_BTW_TILES, IMAGE_CONTAINER_MAX_WIDTH } from 'constants/gallery';
-import { PublicCollectionGalleryContext } from 'utils/publicCollectionGallery';
-import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
 import { DeduplicateContext } from 'pages/deduplicate';
 import { logError } from '@ente/shared/sentry';
 import { Overlay } from '@ente/shared/components/Container';
@@ -21,6 +19,7 @@ import { FILE_TYPE } from 'constants/file';
 import AlbumOutlined from '@mui/icons-material/AlbumOutlined';
 import Avatar from './Avatar';
 import { shouldShowAvatar } from 'utils/file';
+import { CustomError } from '@ente/shared/error';
 
 interface IProps {
     file: EnteFile;
@@ -217,15 +216,8 @@ const Cont = styled('div')<{ disabled: boolean }>`
 
 export default function PreviewCard(props: IProps) {
     const galleryContext = useContext(GalleryContext);
-    const publicCollectionGalleryContext = useContext(
-        PublicCollectionGalleryContext
-    );
     const deduplicateContext = useContext(DeduplicateContext);
 
-    const thumbsStore = publicCollectionGalleryContext?.accessedThroughSharedURL
-        ? publicCollectionGalleryContext.thumbs
-        : galleryContext.thumbs;
-
     const {
         file,
         onClick,
@@ -256,51 +248,21 @@ export default function PreviewCard(props: IProps) {
                 if (file.msrc) {
                     return;
                 }
-                let url: string;
-                // check in in-memory cache
-                if (thumbsStore.has(file.id)) {
-                    url = thumbsStore.get(file.id);
-                } else {
-                    // check in cacheStorage
-                    if (
-                        publicCollectionGalleryContext.accessedThroughSharedURL
-                    ) {
-                        url =
-                            await PublicCollectionDownloadManager.getCachedThumbnail(
-                                file
-                            );
-                    } else {
-                        url = await DownloadManager.getCachedThumbnail(file);
-                    }
-                    if (url) {
-                        thumbsStore.set(file.id, url);
-                    } else {
-                        // download thumbnail
-                        if (props.showPlaceholder) {
-                            return;
-                        }
-                        if (
-                            publicCollectionGalleryContext.accessedThroughSharedURL
-                        ) {
-                            url =
-                                await PublicCollectionDownloadManager.getThumbnail(
-                                    file,
-                                    publicCollectionGalleryContext.token,
-                                    publicCollectionGalleryContext.passwordToken
-                                );
-                        } else {
-                            url = await DownloadManager.getThumbnail(file);
-                        }
-                        thumbsStore.set(file.id, url);
-                    }
-                }
-                if (!isMounted.current) {
+                const url: string =
+                    await DownloadManager.getThumbnailForPreview(
+                        file,
+                        props.showPlaceholder
+                    );
+
+                if (!isMounted.current || !url) {
                     return;
                 }
                 setImgSrc(url);
                 updateURL(file.id, url);
             } catch (e) {
-                logError(e, 'preview card useEffect failed');
+                if (e.message !== CustomError.URL_ALREADY_SET) {
+                    logError(e, 'preview card useEffect failed');
+                }
                 // no-op
             }
         };
@@ -338,7 +300,7 @@ export default function PreviewCard(props: IProps) {
 
     return (
         <Cont
-            key={`thumb-${file.id}-${props.showPlaceholder}`}
+            key={`thumb-${file.id}}`}
             onClick={handleClick}
             onMouseEnter={handleHover}
             disabled={!file?.msrc && !imgSrc}

apps/photos/src/pages/deduplicate/index.tsx

@@ -4,12 +4,8 @@ import PhotoFrame from 'components/PhotoFrame';
 import { ALL_SECTION } from 'constants/collection';
 import { AppContext } from 'pages/_app';
 import { createContext, useContext, useEffect, useState } from 'react';
-import {
-    getDuplicateFiles,
-    clubDuplicatesByTime,
-} from 'services/deduplicationService';
-import { syncFiles, trashFiles } from 'services/fileService';
-import { EnteFile } from 'types/file';
+import { getDuplicates, Duplicate } from 'services/deduplicationService';
+import { getLocalFiles, trashFiles } from 'services/fileService';
 import { SelectedState } from 'types/gallery';
 
 import { ApiError } from '@ente/shared/error';
@@ -24,7 +20,7 @@ import { PHOTOS_PAGES as PAGES } from '@ente/shared/constants/pages';
 import router from 'next/router';
 import { getKey, SESSION_KEYS } from '@ente/shared/storage/sessionStorage';
 import { styled } from '@mui/material';
-import { getLatestCollections } from 'services/collectionService';
+import { getLocalCollections } from 'services/collectionService';
 import EnteSpinner from '@ente/shared/components/EnteSpinner';
 import { VerticallyCentered } from '@ente/shared/components/Container';
 import Typography from '@mui/material/Typography';
@@ -43,9 +39,7 @@ export const Info = styled('div')`
 export default function Deduplicate() {
     const { setDialogMessage, startLoading, finishLoading, showNavBar } =
         useContext(AppContext);
-    const [duplicateFiles, setDuplicateFiles] = useState<EnteFile[]>(null);
-    const [clubSameTimeFilesOnly, setClubSameTimeFilesOnly] = useState(false);
-    const [fileSizeMap, setFileSizeMap] = useState(new Map<number, number>());
+    const [duplicates, setDuplicates] = useState<Duplicate[]>(null);
     const [collectionNameMap, setCollectionNameMap] = useState(
         new Map<number, string>()
     );
@@ -69,31 +63,22 @@ export default function Deduplicate() {
 
     useEffect(() => {
         syncWithRemote();
-    }, [clubSameTimeFilesOnly]);
-
-    const fileToCollectionsMap = useMemoSingleThreaded(() => {
-        return constructFileToCollectionMap(duplicateFiles);
-    }, [duplicateFiles]);
+    }, []);
 
     const syncWithRemote = async () => {
         startLoading();
-        const collections = await getLatestCollections();
+        const collections = await getLocalCollections();
         const collectionNameMap = new Map<number, string>();
         for (const collection of collections) {
             collectionNameMap.set(collection.id, collection.name);
         }
         setCollectionNameMap(collectionNameMap);
-        const files = await syncFiles('normal', collections, () => null);
-        let duplicates = await getDuplicateFiles(files, collectionNameMap);
-        if (clubSameTimeFilesOnly) {
-            duplicates = clubDuplicatesByTime(duplicates);
-        }
+        const files = await getLocalFiles();
+        const duplicateFiles = await getDuplicates(files, collectionNameMap);
         const currFileSizeMap = new Map<number, number>();
-        let allDuplicateFiles: EnteFile[] = [];
         let toSelectFileIDs: number[] = [];
         let count = 0;
-        for (const dupe of duplicates) {
-            allDuplicateFiles = [...allDuplicateFiles, ...dupe.files];
+        for (const dupe of duplicateFiles) {
             // select all except first file
             toSelectFileIDs = [
                 ...toSelectFileIDs,
@@ -105,8 +90,7 @@ export default function Deduplicate() {
                 currFileSizeMap.set(file.id, dupe.size);
             }
         }
-        setDuplicateFiles(allDuplicateFiles);
-        setFileSizeMap(currFileSizeMap);
+        setDuplicates(duplicateFiles);
         const selectedFiles = {
             count: count,
             ownCount: count,
@@ -119,6 +103,16 @@ export default function Deduplicate() {
         finishLoading();
     };
 
+    const duplicateFiles = useMemoSingleThreaded(() => {
+        return (duplicates ?? []).reduce((acc, dupe) => {
+            return [...acc, ...dupe.files];
+        }, []);
+    }, [duplicates]);
+
+    const fileToCollectionsMap = useMemoSingleThreaded(() => {
+        return constructFileToCollectionMap(duplicateFiles);
+    }, [duplicateFiles]);
+
     const deleteFileHelper = async () => {
         try {
             startLoading();
@@ -153,7 +147,7 @@ export default function Deduplicate() {
         setSelected({ count: 0, collectionID: 0, ownCount: 0 });
     };
 
-    if (!duplicateFiles) {
+    if (!duplicates) {
         return (
             <VerticallyCentered>
                 <EnteSpinner />
@@ -166,19 +160,10 @@ export default function Deduplicate() {
             value={{
                 ...DefaultDeduplicateContext,
                 collectionNameMap,
-                clubSameTimeFilesOnly,
-                setClubSameTimeFilesOnly,
-                fileSizeMap,
                 isOnDeduplicatePage: true,
             }}>
             {duplicateFiles.length > 0 && (
-                <Info>
-                    {t('DEDUPLICATE_BASED_ON', {
-                        context: clubSameTimeFilesOnly
-                            ? 'SIZE_AND_CAPTURE_TIME'
-                            : 'SIZE',
-                    })}
-                </Info>
+                <Info>{t('DEDUPLICATE_BASED_ON_SIZE')}</Info>
             )}
             {duplicateFiles.length === 0 ? (
                 <VerticallyCentered>
@@ -188,7 +173,9 @@ export default function Deduplicate() {
                 </VerticallyCentered>
             ) : (
                 <PhotoFrame
+                    page={PAGES.DEDUPLICATE}
                     files={duplicateFiles}
+                    duplicates={duplicates}
                     syncWithRemote={syncWithRemote}
                     setSelected={setSelected}
                     selected={selected}

apps/photos/src/pages/gallery/index.tsx

@@ -129,6 +129,8 @@ import InMemoryStore, { MS_KEYS } from '@ente/shared/storage/InMemoryStore';
 import { syncEmbeddings } from 'services/embeddingService';
 import { ClipService } from 'services/clipService';
 import isElectron from 'is-electron';
+import downloadManager from 'services/download';
+import { APPS } from '@ente/shared/apps/constants';
 
 export const DeadCenter = styled('div')`
     flex: 1;
@@ -140,8 +142,6 @@ export const DeadCenter = styled('div')`
 `;
 
 const defaultGalleryContext: GalleryContextType = {
-    thumbs: new Map(),
-    files: new Map(),
     showPlanSelectorModal: () => null,
     setActiveCollectionID: () => null,
     syncWithRemote: () => null,
@@ -296,7 +296,8 @@ export default function Gallery() {
     useEffect(() => {
         appContext.showNavBar(true);
         const key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
-        if (!key) {
+        const token = getToken();
+        if (!key || !token) {
             InMemoryStore.set(MS_KEYS.REDIRECT_URL, PAGES.GALLERY);
             router.push(PAGES.ROOT);
             return;
@@ -307,6 +308,7 @@ export default function Gallery() {
             if (!valid) {
                 return;
             }
+            await downloadManager.init(APPS.PHOTOS, { token });
             setupSelectAllKeyBoardShortcutHandler();
             setActiveCollectionID(ALL_SECTION);
             setIsFirstLoad(isFirstLogin());
@@ -408,7 +410,7 @@ export default function Gallery() {
     }, [fixCreationTimeAttributes]);
 
     useEffect(() => {
-        if (typeof activeCollectionID === 'undefined') {
+        if (typeof activeCollectionID === 'undefined' || !router.isReady) {
             return;
         }
         let collectionURL = '';
@@ -427,14 +429,8 @@ export default function Gallery() {
             }
         }
         const href = `/gallery${collectionURL}`;
-        const delayRouteChange = () => {
-            setTimeout(() => {
-                router.push(href, undefined, { shallow: true });
-            }, 1000);
-        };
-
-        delayRouteChange();
-    }, [activeCollectionID]);
+        router.push(href, undefined, { shallow: true });
+    }, [activeCollectionID, router.isReady]);
 
     useEffect(() => {
         const key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
@@ -1096,6 +1092,7 @@ export default function Gallery() {
                     <GalleryEmptyState openUploader={openUploader} />
                 ) : (
                     <PhotoFrame
+                        page={PAGES.GALLERY}
                         files={filteredData}
                         syncWithRemote={syncWithRemote}
                         favItemIds={favItemIds}

apps/photos/src/pages/index.tsx

@@ -132,7 +132,11 @@ export default function LandingPage() {
         const user = getData(LS_KEYS.USER);
         let key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
         if (!key && isElectron()) {
-            key = await ElectronAPIs.getEncryptionKey();
+            try {
+                key = await ElectronAPIs.getEncryptionKey();
+            } catch (e) {
+                logError(e, 'getEncryptionKey failed');
+            }
             if (key) {
                 await saveKeyInSessionStore(
                     SESSION_KEYS.ENCRYPTION_KEY,

apps/photos/src/pages/shared-albums/index.tsx

@@ -58,9 +58,8 @@ import MoreHoriz from '@mui/icons-material/MoreHoriz';
 import OverflowMenu from '@ente/shared/components/OverflowMenu/menu';
 import { OverflowMenuOption } from '@ente/shared/components/OverflowMenu/option';
 import { ENTE_WEBSITE_LINK } from '@ente/shared/constants/urls';
-
-const defaultThumbStore = new Map();
-const defaultFileStore = new Map();
+import { APPS } from '@ente/shared/apps/constants';
+import downloadManager from 'services/download';
 
 export default function PublicCollectionGallery() {
     const token = useRef<string>(null);
@@ -156,6 +155,7 @@ export default function PublicCollectionGallery() {
             let redirectingToWebsite = false;
             try {
                 const cryptoWorker = await ComlinkCryptoWorker.getInstance();
+                await downloadManager.init(APPS.ALBUMS);
 
                 url.current = window.location.href;
                 const currentURL = new URL(url.current);
@@ -173,6 +173,7 @@ export default function PublicCollectionGallery() {
                         ? await cryptoWorker.toB64(bs58.decode(ck))
                         : await cryptoWorker.fromHex(ck);
                 token.current = t;
+                downloadManager.updateToken(token.current);
                 collectionKey.current = dck;
                 url.current = window.location.href;
                 const localCollection = await getLocalPublicCollection(
@@ -195,6 +196,10 @@ export default function PublicCollectionGallery() {
                     setPublicFiles(localPublicFiles);
                     passwordJWTToken.current =
                         await getLocalPublicCollectionPassword(collectionUID);
+                    downloadManager.updateToken(
+                        token.current,
+                        passwordJWTToken.current
+                    );
                 }
                 await syncWithRemote();
             } finally {
@@ -218,12 +223,7 @@ export default function PublicCollectionGallery() {
         appContext.startLoading();
         for (const file of publicFiles) {
             try {
-                await downloadFile(
-                    file,
-                    true,
-                    token.current,
-                    passwordJWTToken.current
-                );
+                await downloadFile(file);
             } catch (e) {
                 // do nothing
             }
@@ -385,7 +385,11 @@ export default function PublicCollectionGallery() {
                     hashedPassword
                 );
                 passwordJWTToken.current = jwtToken;
-                savePublicCollectionPassword(collectionUID, jwtToken);
+                downloadManager.updateToken(
+                    token.current,
+                    passwordJWTToken.current
+                );
+                await savePublicCollectionPassword(collectionUID, jwtToken);
             } catch (e) {
                 const parsedError = parseSharingErrorCodes(e);
                 if (parsedError.message === CustomError.TOKEN_EXPIRED) {
@@ -446,8 +450,6 @@ export default function PublicCollectionGallery() {
                 accessedThroughSharedURL: true,
                 photoListHeader,
                 photoListFooter,
-                thumbs: defaultThumbStore,
-                files: defaultFileStore,
             }}>
             <FullScreenDropZone
                 getDragAndDropRootProps={getDragAndDropRootProps}>
@@ -463,6 +465,7 @@ export default function PublicCollectionGallery() {
                     openUploader={openUploader}
                 />
                 <PhotoFrame
+                    page={PAGES.SHARED_ALBUMS}
                     files={publicFiles}
                     syncWithRemote={syncWithRemote}
                     setSelected={() => null}

apps/photos/src/services/clipService.ts

@@ -4,7 +4,7 @@ import {
     getLocalEmbeddings,
 } from './embeddingService';
 import { getAllLocalFiles, getLocalFiles } from './fileService';
-import downloadManager from './downloadManager';
+import downloadManager from './download';
 import { logError } from '@ente/shared/sentry';
 import { addLogLine } from '@ente/shared/logging';
 import { Events, eventBus } from '@ente/shared/events';
@@ -17,7 +17,6 @@ import { getPersonalFiles } from 'utils/file';
 import { FILE_TYPE } from 'constants/file';
 import ComlinkCryptoWorker from '@ente/shared/crypto';
 import { Embedding, Model } from 'types/embedding';
-import { getToken } from '@ente/shared/storage/localStorage/helpers';
 import isElectron from 'is-electron';
 
 const CLIP_EMBEDDING_LENGTH = 512;
@@ -298,19 +297,7 @@ class ClipServiceImpl {
     };
 
     private extractFileClipImageEmbedding = async (file: EnteFile) => {
-        const token = getToken();
-        if (!token) {
-            return;
-        }
-        let thumb: Uint8Array;
-        const thumbURL = await downloadManager.getCachedThumbnail(file);
-        if (thumbURL) {
-            thumb = await fetch(thumbURL)
-                .then((response) => response.arrayBuffer())
-                .then((buffer) => new Uint8Array(buffer));
-        } else {
-            thumb = await downloadManager.downloadThumb(token, file);
-        }
+        const thumb = await downloadManager.getThumbnail(file);
         const embedding = await ElectronAPIs.computeImageEmbedding(thumb);
         return embedding;
     };

apps/photos/src/services/deduplicationService.ts

@@ -16,12 +16,12 @@ interface DuplicatesResponse {
     }>;
 }
 
-interface DuplicateFiles {
+export interface Duplicate {
     files: EnteFile[];
     size: number;
 }
 
-export async function getDuplicateFiles(
+export async function getDuplicates(
     files: EnteFile[],
     collectionNameMap: Map<number, string>
 ) {
@@ -33,7 +33,7 @@ export async function getDuplicateFiles(
             fileMap.set(file.id, file);
         }
 
-        let result: DuplicateFiles[] = [];
+        let result: Duplicate[] = [];
 
         for (const dupe of dupes) {
             let duplicateFiles: EnteFile[] = [];
@@ -64,8 +64,8 @@ export async function getDuplicateFiles(
     }
 }
 
-function getDupesGroupedBySameFileHashes(dupe: DuplicateFiles) {
-    const result: DuplicateFiles[] = [];
+function getDupesGroupedBySameFileHashes(dupe: Duplicate) {
+    const result: Duplicate[] = [];
 
     const fileWithHashes: EnteFile[] = [];
     const fileWithoutHashes: EnteFile[] = [];
@@ -95,8 +95,8 @@ function getDupesGroupedBySameFileHashes(dupe: DuplicateFiles) {
     return result;
 }
 
-function groupDupesByFileHashes(dupe: DuplicateFiles) {
-    const result: DuplicateFiles[] = [];
+function groupDupesByFileHashes(dupe: Duplicate) {
+    const result: Duplicate[] = [];
 
     const filesSortedByFileHash = dupe.files
         .map((file) => {
@@ -141,51 +141,6 @@ function groupDupesByFileHashes(dupe: DuplicateFiles) {
     return result;
 }
 
-export function clubDuplicatesByTime(dupes: DuplicateFiles[]) {
-    const result: DuplicateFiles[] = [];
-    for (const dupe of dupes) {
-        let files: EnteFile[] = [];
-        const creationTimeCounter = new Map<number, number>();
-
-        let mostFreqCreationTime = 0;
-        let mostFreqCreationTimeCount = 0;
-        for (const file of dupe.files) {
-            const creationTime = file.metadata.creationTime;
-            if (creationTimeCounter.has(creationTime)) {
-                creationTimeCounter.set(
-                    creationTime,
-                    creationTimeCounter.get(creationTime) + 1
-                );
-            } else {
-                creationTimeCounter.set(creationTime, 1);
-            }
-            if (
-                creationTimeCounter.get(creationTime) >
-                mostFreqCreationTimeCount
-            ) {
-                mostFreqCreationTime = creationTime;
-                mostFreqCreationTimeCount =
-                    creationTimeCounter.get(creationTime);
-            }
-
-            files.push(file);
-        }
-
-        files = files.filter((file) => {
-            return file.metadata.creationTime === mostFreqCreationTime;
-        });
-
-        if (files.length > 1) {
-            result.push({
-                files,
-                size: dupe.size,
-            });
-        }
-    }
-
-    return result;
-}
-
 async function fetchDuplicateFileIDs() {
     try {
         const response = await HTTPService.get(

apps/photos/src/services/download/clients/photos.ts

@@ -0,0 +1,73 @@
+import HTTPService from '@ente/shared/network/HTTPService';
+import { getFileURL, getThumbnailURL } from '@ente/shared/network/api';
+import { EnteFile } from 'types/file';
+import { DownloadClient } from 'services/download';
+import { CustomError } from '@ente/shared/error';
+import { retryAsyncFunction } from 'utils/network';
+
+export class PhotosDownloadClient implements DownloadClient {
+    constructor(private token: string, private timeout: number) {}
+    updateTokens(token: string) {
+        this.token = token;
+    }
+
+    updateTimeout(timeout: number) {
+        this.timeout = timeout;
+    }
+
+    async downloadThumbnail(file: EnteFile): Promise<Uint8Array> {
+        if (!this.token) {
+            throw Error(CustomError.TOKEN_MISSING);
+        }
+        const resp = await retryAsyncFunction(() =>
+            HTTPService.get(
+                getThumbnailURL(file.id),
+                null,
+                { 'X-Auth-Token': this.token },
+                { responseType: 'arraybuffer', timeout: this.timeout }
+            )
+        );
+        if (typeof resp.data === 'undefined') {
+            throw Error(CustomError.REQUEST_FAILED);
+        }
+        return new Uint8Array(resp.data);
+    }
+
+    async downloadFile(
+        file: EnteFile,
+        onDownloadProgress: (event: { loaded: number; total: number }) => void
+    ): Promise<Uint8Array> {
+        if (!this.token) {
+            throw Error(CustomError.TOKEN_MISSING);
+        }
+        const resp = await retryAsyncFunction(() =>
+            HTTPService.get(
+                getFileURL(file.id),
+                null,
+                { 'X-Auth-Token': this.token },
+                {
+                    responseType: 'arraybuffer',
+                    timeout: this.timeout,
+                    onDownloadProgress,
+                }
+            )
+        );
+        if (typeof resp.data === 'undefined') {
+            throw Error(CustomError.REQUEST_FAILED);
+        }
+        return new Uint8Array(resp.data);
+    }
+
+    async downloadFileStream(file: EnteFile): Promise<Response> {
+        if (!this.token) {
+            throw Error(CustomError.TOKEN_MISSING);
+        }
+        return retryAsyncFunction(() =>
+            fetch(getFileURL(file.id), {
+                headers: {
+                    'X-Auth-Token': this.token,
+                },
+            })
+        );
+    }
+}

apps/photos/src/services/download/clients/publicAlbums.ts

@@ -0,0 +1,95 @@
+import HTTPService from '@ente/shared/network/HTTPService';
+import {
+    getPublicCollectionFileURL,
+    getPublicCollectionThumbnailURL,
+} from '@ente/shared/network/api';
+import { EnteFile } from 'types/file';
+import { DownloadClient } from 'services/download';
+import { CustomError } from '@ente/shared/error';
+import { retryAsyncFunction } from 'utils/network';
+
+export class PublicAlbumsDownloadClient implements DownloadClient {
+    constructor(
+        private token: string,
+        private passwordToken: string,
+        private timeout: number
+    ) {}
+
+    updateTokens(token: string, passwordToken: string) {
+        this.token = token;
+        this.passwordToken = passwordToken;
+    }
+
+    updateTimeout(timeout: number) {
+        this.timeout = timeout;
+    }
+
+    downloadThumbnail = async (file: EnteFile) => {
+        if (!this.token) {
+            throw Error(CustomError.TOKEN_MISSING);
+        }
+        const resp = await HTTPService.get(
+            getPublicCollectionThumbnailURL(file.id),
+            null,
+            {
+                'X-Auth-Access-Token': this.token,
+                ...(this.passwordToken && {
+                    'X-Auth-Access-Token-JWT': this.passwordToken,
+                }),
+            },
+            { responseType: 'arraybuffer' }
+        );
+
+        if (typeof resp.data === 'undefined') {
+            throw Error(CustomError.REQUEST_FAILED);
+        }
+        return new Uint8Array(resp.data);
+    };
+
+    downloadFile = async (
+        file: EnteFile,
+        onDownloadProgress: (event: { loaded: number; total: number }) => void
+    ) => {
+        if (!this.token) {
+            throw Error(CustomError.TOKEN_MISSING);
+        }
+        const resp = await retryAsyncFunction(() =>
+            HTTPService.get(
+                getPublicCollectionFileURL(file.id),
+                null,
+                {
+                    'X-Auth-Access-Token': this.token,
+                    ...(this.passwordToken && {
+                        'X-Auth-Access-Token-JWT': this.passwordToken,
+                    }),
+                },
+                {
+                    responseType: 'arraybuffer',
+                    timeout: this.timeout,
+                    onDownloadProgress,
+                }
+            )
+        );
+
+        if (typeof resp.data === 'undefined') {
+            throw Error(CustomError.REQUEST_FAILED);
+        }
+        return new Uint8Array(resp.data);
+    };
+
+    async downloadFileStream(file: EnteFile): Promise<Response> {
+        if (!this.token) {
+            throw Error(CustomError.TOKEN_MISSING);
+        }
+        return retryAsyncFunction(() =>
+            fetch(getPublicCollectionFileURL(file.id), {
+                headers: {
+                    'X-Auth-Access-Token': this.token,
+                    ...(this.passwordToken && {
+                        'X-Auth-Access-Token-JWT': this.passwordToken,
+                    }),
+                },
+            })
+        );
+    }
+}

apps/photos/src/services/downloadManager.ts → apps/photos/src/services/download/index.ts

@@ -1,11 +1,7 @@
-import { getToken } from '@ente/shared/storage/localStorage/helpers';
-import { getFileURL, getThumbnailURL } from '@ente/shared/network/api';
 import {
     generateStreamFromArrayBuffer,
     getRenderableFileURL,
-    createTypedObjectURL,
 } from 'utils/file';
-import HTTPService from '@ente/shared/network/HTTPService';
 import { EnteFile } from 'types/file';
 
 import { logError } from '@ente/shared/sentry';
@@ -17,173 +13,282 @@ import { CACHES } from '@ente/shared/storage/cacheStorage/constants';
 import { Remote } from 'comlink';
 import { DedicatedCryptoWorker } from '@ente/shared/crypto/internal/crypto.worker';
 import { LimitedCache } from '@ente/shared/storage/cacheStorage/types';
-import { retryAsyncFunction } from 'utils/network';
 import { addLogLine } from '@ente/shared/logging';
+import { APPS } from '@ente/shared/apps/constants';
+import { PhotosDownloadClient } from './clients/photos';
+import { PublicAlbumsDownloadClient } from './clients/publicAlbums';
+import isElectron from 'is-electron';
+import { isInternalUser } from 'utils/user';
 
-class DownloadManager {
-    private fileObjectURLPromise = new Map<
-        string,
-        Promise<{ original: string[]; converted: string[] }>
-    >();
-    private thumbnailObjectURLPromise = new Map<number, Promise<string>>();
+export type LivePhotoSourceURL = {
+    image: () => Promise<string>;
+    video: () => Promise<string>;
+};
+
+export type LoadedLivePhotoSourceURL = {
+    image: string;
+    video: string;
+};
+
+export type SourceURLs = {
+    url: string | LivePhotoSourceURL | LoadedLivePhotoSourceURL;
+    isOriginal: boolean;
+    isRenderable: boolean;
+    type: 'normal' | 'livePhoto';
+};
+
+export type OnDownloadProgress = (event: {
+    loaded: number;
+    total: number;
+}) => void;
+
+export interface DownloadClient {
+    updateTokens: (token: string, passwordToken?: string) => void;
+    updateTimeout: (timeout: number) => void;
+    downloadThumbnail: (
+        file: EnteFile,
+        timeout?: number
+    ) => Promise<Uint8Array>;
+    downloadFile: (
+        file: EnteFile,
+        onDownloadProgress: OnDownloadProgress
+    ) => Promise<Uint8Array>;
+    downloadFileStream: (file: EnteFile) => Promise<Response>;
+}
+
+const FILE_CACHE_LIMIT = 5 * 1024 * 1024 * 1024; // 5GB
+
+class DownloadManagerImpl {
+    private ready: boolean = false;
+    private downloadClient: DownloadClient;
+    private thumbnailCache?: LimitedCache;
+    // disk cache is only available on electron
+    private diskFileCache?: LimitedCache;
+    private cryptoWorker: Remote<DedicatedCryptoWorker>;
+
+    private fileObjectURLPromises = new Map<number, Promise<SourceURLs>>();
+    private fileConversionPromises = new Map<number, Promise<SourceURLs>>();
+    private thumbnailObjectURLPromises = new Map<number, Promise<string>>();
 
     private fileDownloadProgress = new Map<number, number>();
 
     private progressUpdater: (value: Map<number, number>) => void = () => {};
 
-    private thumbnailCache: LimitedCache;
+    async init(
+        app: APPS,
+        tokens?: { token: string; passwordToken?: string } | { token: string },
+        timeout?: number
+    ) {
+        try {
+            if (this.ready) {
+                addLogLine('DownloadManager already initialized');
+                return;
+            }
+            this.downloadClient = createDownloadClient(app, tokens, timeout);
+            this.thumbnailCache = await openThumbnailCache();
+            this.diskFileCache = isElectron() && (await openDiskFileCache());
+            this.cryptoWorker = await ComlinkCryptoWorker.getInstance();
+            this.ready = true;
+        } catch (e) {
+            logError(e, 'DownloadManager init failed');
+            throw e;
+        }
+    }
+
+    updateToken(token: string, passwordToken?: string) {
+        this.downloadClient.updateTokens(token, passwordToken);
+    }
+
+    updateCryptoWorker(cryptoWorker: Remote<DedicatedCryptoWorker>) {
+        this.cryptoWorker = cryptoWorker;
+    }
+
+    updateTimeout(timeout: number) {
+        this.downloadClient.updateTimeout(timeout);
+    }
 
     setProgressUpdater(progressUpdater: (value: Map<number, number>) => void) {
         this.progressUpdater = progressUpdater;
     }
 
-    private async getThumbnailCache() {
-        try {
-            if (!this.thumbnailCache) {
-                this.thumbnailCache = await CacheStorageService.open(
-                    CACHES.THUMBS
-                );
-            }
-            return this.thumbnailCache;
-        } catch (e) {
-            return null;
-            // ignore
-        }
+    async reloadCaches() {
+        this.thumbnailCache = await openThumbnailCache();
+        this.diskFileCache = isElectron() && (await openDiskFileCache());
     }
 
-    public async getCachedThumbnail(file: EnteFile) {
+    private async getCachedThumbnail(fileID: number) {
         try {
-            const thumbnailCache = await this.getThumbnailCache();
-            const cacheResp: Response = await thumbnailCache?.match(
-                file.id.toString()
+            const cacheResp: Response = await this.thumbnailCache?.match(
+                fileID.toString()
             );
 
             if (cacheResp) {
-                return URL.createObjectURL(await cacheResp.blob());
+                return new Uint8Array(await cacheResp.arrayBuffer());
             }
-            return null;
         } catch (e) {
             logError(e, 'failed to get cached thumbnail');
             throw e;
         }
     }
-
-    public async getThumbnail(
-        file: EnteFile,
-        tokenOverride?: string,
-        usingWorker?: Remote<DedicatedCryptoWorker>,
-        timeout?: number
-    ) {
+    private async getCachedFile(file: EnteFile): Promise<Response> {
         try {
-            const token = tokenOverride || getToken();
-            if (!token) {
+            if (!this.diskFileCache) {
                 return null;
             }
-            if (!this.thumbnailObjectURLPromise.has(file.id)) {
-                const downloadPromise = async () => {
-                    const thumbnailCache = await this.getThumbnailCache();
-                    const cachedThumb = await this.getCachedThumbnail(file);
-                    if (cachedThumb) {
-                        return cachedThumb;
-                    }
-                    const thumb = await this.downloadThumb(
-                        token,
-                        file,
-                        usingWorker,
-                        timeout
-                    );
-                    const thumbBlob = new Blob([thumb]);
+            const cacheResp: Response = await this.diskFileCache?.match(
+                file.id.toString(),
+                { sizeInBytes: file.info?.fileSize }
+            );
+            return cacheResp?.clone();
+        } catch (e) {
+            logError(e, 'failed to get cached file');
+            throw e;
+        }
+    }
 
-                    thumbnailCache
-                        ?.put(file.id.toString(), new Response(thumbBlob))
-                        .catch((e) => {
-                            logError(e, 'cache put failed');
-                            // TODO: handle storage full exception.
-                        });
-                    return URL.createObjectURL(thumbBlob);
-                };
-                this.thumbnailObjectURLPromise.set(file.id, downloadPromise());
+    private downloadThumb = async (file: EnteFile) => {
+        const encrypted = await this.downloadClient.downloadThumbnail(file);
+        const decrypted = await this.cryptoWorker.decryptThumbnail(
+            encrypted,
+            await this.cryptoWorker.fromB64(file.thumbnail.decryptionHeader),
+            file.key
+        );
+        return decrypted;
+    };
+
+    async getThumbnail(file: EnteFile, localOnly = false) {
+        try {
+            if (!this.ready) {
+                throw Error(CustomError.DOWNLOAD_MANAGER_NOT_READY);
+            }
+            const cachedThumb = await this.getCachedThumbnail(file.id);
+            if (cachedThumb) {
+                return cachedThumb;
+            }
+            if (localOnly) {
+                return null;
             }
+            const thumb = await this.downloadThumb(file);
+
+            this.thumbnailCache
+                ?.put(file.id.toString(), new Response(thumb))
+                .catch((e) => {
+                    logError(e, 'thumb cache put failed');
+                    // TODO: handle storage full exception.
+                });
+            return thumb;
+        } catch (e) {
+            logError(e, 'getThumbnail failed');
+            throw e;
+        }
+    }
 
-            return await this.thumbnailObjectURLPromise.get(file.id);
+    async getThumbnailForPreview(file: EnteFile, localOnly = false) {
+        try {
+            if (!this.ready) {
+                throw Error(CustomError.DOWNLOAD_MANAGER_NOT_READY);
+            }
+            if (!this.thumbnailObjectURLPromises.has(file.id)) {
+                const thumbPromise = this.getThumbnail(file, localOnly);
+                const thumbURLPromise = thumbPromise.then(
+                    (thumb) => thumb && URL.createObjectURL(new Blob([thumb]))
+                );
+                this.thumbnailObjectURLPromises.set(file.id, thumbURLPromise);
+            }
+            let thumb = await this.thumbnailObjectURLPromises.get(file.id);
+            if (!thumb && !localOnly) {
+                this.thumbnailObjectURLPromises.delete(file.id);
+                thumb = await this.getThumbnailForPreview(file, localOnly);
+            }
+            return thumb;
         } catch (e) {
-            this.thumbnailObjectURLPromise.delete(file.id);
+            this.thumbnailObjectURLPromises.delete(file.id);
             logError(e, 'get DownloadManager preview Failed');
             throw e;
         }
     }
 
-    downloadThumb = async (
-        token: string,
+    getFileForPreview = async (
         file: EnteFile,
-        usingWorker?: Remote<DedicatedCryptoWorker>,
-        timeout?: number
-    ) => {
-        const resp = await HTTPService.get(
-            getThumbnailURL(file.id),
-            null,
-            { 'X-Auth-Token': token },
-            { responseType: 'arraybuffer', timeout }
-        );
-        if (typeof resp.data === 'undefined') {
-            throw Error(CustomError.REQUEST_FAILED);
+        forceConvert = false
+    ): Promise<SourceURLs> => {
+        try {
+            if (!this.ready) {
+                throw Error(CustomError.DOWNLOAD_MANAGER_NOT_READY);
+            }
+            const getFileForPreviewPromise = async () => {
+                const fileBlob = await new Response(
+                    await this.getFile(file, true)
+                ).blob();
+                const { url: originalFileURL } =
+                    await this.fileObjectURLPromises.get(file.id);
+
+                const converted = await getRenderableFileURL(
+                    file,
+                    fileBlob,
+                    originalFileURL as string,
+                    forceConvert
+                );
+                return converted;
+            };
+            if (forceConvert || !this.fileConversionPromises.has(file.id)) {
+                this.fileConversionPromises.set(
+                    file.id,
+                    getFileForPreviewPromise()
+                );
+            }
+            const fileURLs = await this.fileConversionPromises.get(file.id);
+            return fileURLs;
+        } catch (e) {
+            this.fileConversionPromises.delete(file.id);
+            logError(e, 'download manager getFileForPreview Failed');
+            throw e;
         }
-        const cryptoWorker =
-            usingWorker || (await ComlinkCryptoWorker.getInstance());
-        const decrypted = await cryptoWorker.decryptThumbnail(
-            new Uint8Array(resp.data),
-            await cryptoWorker.fromB64(file.thumbnail.decryptionHeader),
-            file.key
-        );
-        return decrypted;
     };
 
-    getFile = async (file: EnteFile, forPreview = false) => {
-        const fileKey = forPreview ? `${file.id}_preview` : `${file.id}`;
+    async getFile(
+        file: EnteFile,
+        cacheInMemory = false
+    ): Promise<ReadableStream<Uint8Array>> {
         try {
-            const getFilePromise = async () => {
+            if (!this.ready) {
+                throw Error(CustomError.DOWNLOAD_MANAGER_NOT_READY);
+            }
+            const getFilePromise = async (): Promise<SourceURLs> => {
                 const fileStream = await this.downloadFile(file);
                 const fileBlob = await new Response(fileStream).blob();
-                if (forPreview) {
-                    return await getRenderableFileURL(file, fileBlob);
-                } else {
-                    const fileURL = await createTypedObjectURL(
-                        fileBlob,
-                        file.metadata.title
-                    );
-                    return { converted: [fileURL], original: [fileURL] };
-                }
+                return {
+                    url: URL.createObjectURL(fileBlob),
+                    isOriginal: true,
+                    isRenderable: false,
+                    type: 'normal',
+                };
             };
-            if (!this.fileObjectURLPromise.get(fileKey)) {
-                this.fileObjectURLPromise.set(fileKey, getFilePromise());
+            if (!this.fileObjectURLPromises.has(file.id)) {
+                if (!cacheInMemory) {
+                    return await this.downloadFile(file);
+                }
+                this.fileObjectURLPromises.set(file.id, getFilePromise());
+            }
+            const fileURLs = await this.fileObjectURLPromises.get(file.id);
+            if (fileURLs.isOriginal) {
+                const fileStream = (await fetch(fileURLs.url as string)).body;
+                return fileStream;
+            } else {
+                return await this.downloadFile(file);
             }
-            const fileURLs = await this.fileObjectURLPromise.get(fileKey);
-            return fileURLs;
         } catch (e) {
-            this.fileObjectURLPromise.delete(fileKey);
-            logError(e, 'download manager Failed to get File');
+            this.fileObjectURLPromises.delete(file.id);
+            logError(e, 'download manager getFile Failed');
             throw e;
         }
-    };
-
-    public async getCachedOriginalFile(file: EnteFile) {
-        return (await this.fileObjectURLPromise.get(file.id.toString()))
-            ?.original;
     }
 
-    async downloadFile(
-        file: EnteFile,
-        tokenOverride?: string,
-        usingWorker?: Remote<DedicatedCryptoWorker>,
-        timeout?: number
-    ) {
+    private async downloadFile(
+        file: EnteFile
+    ): Promise<ReadableStream<Uint8Array>> {
         try {
-            const cryptoWorker =
-                usingWorker || (await ComlinkCryptoWorker.getInstance());
-            const token = tokenOverride || getToken();
-            if (!token) {
-                return null;
-            }
+            addLogLine(`download attempted for fileID:${file.id}`);
             const onDownloadProgress = this.trackDownloadProgress(
                 file.id,
                 file.info?.fileSize
@@ -192,26 +297,30 @@ class DownloadManager {
                 file.metadata.fileType === FILE_TYPE.IMAGE ||
                 file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
             ) {
-                const resp = await retryAsyncFunction(() =>
-                    HTTPService.get(
-                        getFileURL(file.id),
-                        null,
-                        { 'X-Auth-Token': token },
-                        {
-                            responseType: 'arraybuffer',
-                            timeout,
-                            onDownloadProgress,
-                        }
-                    )
-                );
-                this.clearDownloadProgress(file.id);
-                if (typeof resp.data === 'undefined') {
-                    throw Error(CustomError.REQUEST_FAILED);
+                let encrypted = await this.getCachedFile(file);
+                if (!encrypted) {
+                    encrypted = new Response(
+                        await this.downloadClient.downloadFile(
+                            file,
+                            onDownloadProgress
+                        )
+                    );
+                    if (this.diskFileCache) {
+                        this.diskFileCache
+                            .put(file.id.toString(), encrypted.clone())
+                            .catch((e) => {
+                                logError(e, 'file cache put failed');
+                                // TODO: handle storage full exception.
+                            });
+                    }
                 }
+                this.clearDownloadProgress(file.id);
                 try {
-                    const decrypted = await cryptoWorker.decryptFile(
-                        new Uint8Array(resp.data),
-                        await cryptoWorker.fromB64(file.file.decryptionHeader),
+                    const decrypted = await this.cryptoWorker.decryptFile(
+                        new Uint8Array(await encrypted.arrayBuffer()),
+                        await this.cryptoWorker.fromB64(
+                            file.file.decryptionHeader
+                        ),
                         file.key
                     );
                     return generateStreamFromArrayBuffer(decrypted);
@@ -231,27 +340,35 @@ class DownloadManager {
                     throw e;
                 }
             }
-            const resp = await retryAsyncFunction(() =>
-                fetch(getFileURL(file.id), {
-                    headers: {
-                        'X-Auth-Token': token,
-                    },
-                })
-            );
+
+            let resp: Response = await this.getCachedFile(file);
+            if (!resp) {
+                resp = await this.downloadClient.downloadFileStream(file);
+                if (this.diskFileCache) {
+                    this.diskFileCache
+                        .put(file.id.toString(), resp.clone())
+                        .catch((e) => {
+                            logError(e, 'file cache put failed');
+                        });
+                }
+            }
             const reader = resp.body.getReader();
 
             const contentLength = +resp.headers.get('Content-Length') ?? 0;
             let downloadedBytes = 0;
 
             const stream = new ReadableStream({
-                async start(controller) {
+                start: async (controller) => {
                     try {
-                        const decryptionHeader = await cryptoWorker.fromB64(
-                            file.file.decryptionHeader
+                        const decryptionHeader =
+                            await this.cryptoWorker.fromB64(
+                                file.file.decryptionHeader
+                            );
+                        const fileKey = await this.cryptoWorker.fromB64(
+                            file.key
                         );
-                        const fileKey = await cryptoWorker.fromB64(file.key);
                         const { pullState, decryptionChunkSize } =
-                            await cryptoWorker.initChunkDecryption(
+                            await this.cryptoWorker.initChunkDecryption(
                                 decryptionHeader,
                                 fileKey
                             );
@@ -285,7 +402,7 @@ class DownloadManager {
                                             );
                                             try {
                                                 const { decryptedData } =
-                                                    await cryptoWorker.decryptFileChunk(
+                                                    await this.cryptoWorker.decryptFileChunk(
                                                         fileData,
                                                         pullState
                                                     );
@@ -329,7 +446,7 @@ class DownloadManager {
                                         if (data) {
                                             try {
                                                 const { decryptedData } =
-                                                    await cryptoWorker.decryptFileChunk(
+                                                    await this.cryptoWorker.decryptFileChunk(
                                                         data,
                                                         pullState
                                                     );
@@ -412,4 +529,58 @@ class DownloadManager {
     };
 }
 
-export default new DownloadManager();
+const DownloadManager = new DownloadManagerImpl();
+
+export default DownloadManager;
+
+async function openThumbnailCache() {
+    try {
+        return await CacheStorageService.open(CACHES.THUMBS);
+    } catch (e) {
+        logError(e, 'Failed to open thumbnail cache');
+        if (isInternalUser()) {
+            throw e;
+        } else {
+            return null;
+        }
+    }
+}
+
+async function openDiskFileCache() {
+    try {
+        if (!isElectron()) {
+            throw Error(CustomError.NOT_AVAILABLE_ON_WEB);
+        }
+        return await CacheStorageService.open(CACHES.FILES, FILE_CACHE_LIMIT);
+    } catch (e) {
+        logError(e, 'Failed to open file cache');
+        if (isInternalUser()) {
+            throw e;
+        } else {
+            return null;
+        }
+    }
+}
+
+function createDownloadClient(
+    app: APPS,
+    tokens?: { token: string; passwordToken?: string } | { token: string },
+    timeout?: number
+): DownloadClient {
+    if (!timeout) {
+        timeout = 300000; // 5 minute
+    }
+    if (app === APPS.ALBUMS) {
+        if (!tokens) {
+            tokens = { token: undefined, passwordToken: undefined };
+        }
+        const { token, passwordToken } = tokens as {
+            token: string;
+            passwordToken: string;
+        };
+        return new PublicAlbumsDownloadClient(token, passwordToken, timeout);
+    } else {
+        const { token } = tokens;
+        return new PhotosDownloadClient(token, timeout);
+    }
+}

apps/photos/src/services/export/index.ts

@@ -25,7 +25,7 @@ import {
 import { logError } from '@ente/shared/sentry';
 import { getData, LS_KEYS, setData } from '@ente/shared/storage/localStorage';
 import { getAllLocalCollections } from '../collectionService';
-import downloadManager from '../downloadManager';
+import downloadManager from '../download';
 import { getAllLocalFiles } from '../fileService';
 import { EnteFile } from 'types/file';
 
@@ -175,14 +175,6 @@ class ExportService {
         }
     }
 
-    async openExportDirectory(exportFolder: string) {
-        try {
-            await ElectronAPIs.openDirectory(exportFolder);
-        } catch (e) {
-            logError(e, 'openExportDirectory failed');
-        }
-    }
-
     enableContinuousExport() {
         try {
             if (this.continuousExportEventHandler) {
@@ -1061,7 +1053,7 @@ class ExportService {
     ): Promise<void> {
         try {
             const fileUID = getExportRecordFileUID(file);
-            const originalFileStream = await downloadManager.downloadFile(file);
+            const originalFileStream = await downloadManager.getFile(file);
             if (!this.fileReader) {
                 this.fileReader = new FileReader();
             }

apps/photos/src/services/export/migration.ts

@@ -43,7 +43,7 @@ import {
 } from 'utils/export/migration';
 import { FILE_TYPE } from 'constants/file';
 import { decodeLivePhoto } from 'services/livePhotoService';
-import downloadManager from 'services/downloadManager';
+import downloadManager from 'services/download';
 import { sleep } from 'utils/common';
 
 export async function migrateExport(
@@ -343,7 +343,7 @@ async function getFileExportNamesFromExportedFiles(
             For Live Photos we need to download the file to get the image and video name
         */
         if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
-            const fileStream = await downloadManager.downloadFile(file);
+            const fileStream = await downloadManager.getFile(file);
             const fileBlob = await new Response(fileStream).blob();
             const livePhoto = await decodeLivePhoto(file, fileBlob);
             const imageExportName = getUniqueFileExportNameForMigration(

apps/photos/src/services/imageProcessor.ts

@@ -4,16 +4,18 @@ import { logError } from '@ente/shared/sentry';
 import { ElectronFile } from 'types/upload';
 import { CustomError } from '@ente/shared/error';
 import { convertBytesToHumanReadable } from '@ente/shared/utils/size';
+import { WorkerSafeElectronService } from '@ente/shared/electron/service';
 
 class ElectronImageProcessorService {
     async convertToJPEG(fileBlob: Blob, filename: string): Promise<Blob> {
         try {
             const startTime = Date.now();
             const inputFileData = new Uint8Array(await fileBlob.arrayBuffer());
-            const convertedFileData = await ElectronAPIs.convertToJPEG(
-                inputFileData,
-                filename
-            );
+            const convertedFileData =
+                await WorkerSafeElectronService.convertToJPEG(
+                    inputFileData,
+                    filename
+                );
             addLogLine(
                 `originalFileSize:${convertBytesToHumanReadable(
                     fileBlob?.size

apps/photos/src/services/machineLearning/faceService.ts

@@ -10,10 +10,13 @@ import {
     getFaceId,
     areFaceIdsSame,
     extractFaceImages,
+    getLocalFile,
+    getOriginalImageBitmap,
 } from 'utils/machineLearning';
 import { storeFaceCrop } from 'utils/machineLearning/faceCrop';
 import mlIDbStorage from 'utils/storage/mlIDbStorage';
 import ReaderService from './readerService';
+import { imageBitmapToBlob } from 'utils/image';
 
 class FaceService {
     async syncFileFaceDetections(
@@ -184,7 +187,9 @@ class FaceService {
             faceCrop,
             syncContext.config.faceCrop.blobOptions
         );
+        const blob = await imageBitmapToBlob(faceCrop.image);
         faceCrop.image.close();
+        return blob;
     }
 
     async getAllSyncedFacesMap(syncContext: MLSyncContext) {
@@ -234,6 +239,21 @@ class FaceService {
         //     noise: syncContext.faceClusteringResults.noise,
         // };
     }
+
+    public async regenerateFaceCrop(
+        syncContext: MLSyncContext,
+        faceID: string
+    ) {
+        const fileID = Number(faceID.split('-')[0]);
+        const personFace = await mlIDbStorage.getFace(fileID, faceID);
+        if (!personFace) {
+            throw Error('Face not found');
+        }
+
+        const file = await getLocalFile(personFace.fileId);
+        const imageBitmap = await getOriginalImageBitmap(file);
+        return await this.saveFaceCrop(imageBitmap, personFace, syncContext);
+    }
 }
 
 export default new FaceService();

apps/photos/src/services/machineLearning/machineLearningService.ts

@@ -33,6 +33,9 @@ import ObjectService from './objectService';
 import ReaderService from './readerService';
 import { logError } from '@ente/shared/sentry';
 import { addLogLine } from '@ente/shared/logging';
+import downloadManager from 'services/download';
+import { APPS } from '@ente/shared/apps/constants';
+
 class MachineLearningService {
     private initialized = false;
     // private faceDetectionService: FaceDetectionService;
@@ -60,6 +63,7 @@ class MachineLearningService {
             throw Error('Token needed by ml service to sync file');
         }
 
+        await downloadManager.init(APPS.PHOTOS, { token });
         // await this.init();
 
         // Used to debug tf memory leak, all tf memory
@@ -112,6 +116,16 @@ class MachineLearningService {
         return mlSyncResult;
     }
 
+    public async regenerateFaceCrop(
+        token: string,
+        userID: number,
+        faceID: string
+    ) {
+        await downloadManager.init(APPS.PHOTOS, { token });
+        const syncContext = await this.getSyncContext(token, userID);
+        return FaceService.regenerateFaceCrop(syncContext, faceID);
+    }
+
     private newMlData(fileId: number) {
         return {
             fileId,

apps/photos/src/services/machineLearning/mlWorkManager.ts

@@ -1,4 +1,4 @@
-import debounce from 'debounce-promise';
+import debounce from 'debounce';
 import PQueue from 'p-queue';
 import { eventBus, Events } from '@ente/shared/events';
 import { EnteFile } from 'types/file';
@@ -201,35 +201,39 @@ class MLWorkManager {
     }
 
     private async runMLSyncJob(): Promise<MLSyncJobResult> {
-        // TODO: skipping is not required if we are caching chunks through service worker
-        // currently worker chunk itself is not loaded when network is not there
-        if (!navigator.onLine) {
-            addLogLine(
-                'Skipping ml-sync job run as not connected to internet.'
-            );
-            return {
-                shouldBackoff: true,
-                mlSyncResult: undefined,
-            };
-        }
+        try {
+            // TODO: skipping is not required if we are caching chunks through service worker
+            // currently worker chunk itself is not loaded when network is not there
+            if (!navigator.onLine) {
+                addLogLine(
+                    'Skipping ml-sync job run as not connected to internet.'
+                );
+                return {
+                    shouldBackoff: true,
+                    mlSyncResult: undefined,
+                };
+            }
 
-        const token = getToken();
-        const userID = getUserID();
-        const jobWorkerProxy = await this.getSyncJobWorker();
+            const token = getToken();
+            const userID = getUserID();
+            const jobWorkerProxy = await this.getSyncJobWorker();
 
-        const mlSyncResult = await jobWorkerProxy.sync(token, userID);
+            const mlSyncResult = await jobWorkerProxy.sync(token, userID);
 
-        // this.terminateSyncJobWorker();
-        const jobResult: MLSyncJobResult = {
-            shouldBackoff:
-                !!mlSyncResult.error || mlSyncResult.nOutOfSyncFiles < 1,
-            mlSyncResult,
-        };
-        addLogLine('ML Sync Job result: ', JSON.stringify(jobResult));
+            // this.terminateSyncJobWorker();
+            const jobResult: MLSyncJobResult = {
+                shouldBackoff:
+                    !!mlSyncResult.error || mlSyncResult.nOutOfSyncFiles < 1,
+                mlSyncResult,
+            };
+            addLogLine('ML Sync Job result: ', JSON.stringify(jobResult));
 
-        // TODO: redirect/refresh to gallery in case of session_expired, stop ml sync job
+            // TODO: redirect/refresh to gallery in case of session_expired, stop ml sync job
 
-        return jobResult;
+            return jobResult;
+        } catch (e) {
+            logError(e, 'Failed to run MLSync Job');
+        }
     }
 
     public async startSyncJob() {

apps/photos/src/services/machineLearning/peopleService.ts

@@ -64,10 +64,7 @@ class PeopleService {
 
             if (personFace && !personFace.crop?.imageUrl) {
                 const file = await getLocalFile(personFace.fileId);
-                const imageBitmap = await getOriginalImageBitmap(
-                    file,
-                    syncContext.token
-                );
+                const imageBitmap = await getOriginalImageBitmap(file);
                 await FaceService.saveFaceCrop(
                     imageBitmap,
                     personFace,

apps/photos/src/services/machineLearning/readerService.ts

@@ -36,15 +36,11 @@ class ReaderService {
                 )
             ) {
                 fileContext.imageBitmap = await getOriginalImageBitmap(
-                    fileContext.enteFile,
-                    syncContext.token,
-                    await syncContext.getEnteWorker(fileContext.enteFile.id)
+                    fileContext.enteFile
                 );
             } else {
                 fileContext.imageBitmap = await getThumbnailImageBitmap(
-                    fileContext.enteFile,
-                    syncContext.token,
-                    await syncContext.getEnteWorker(fileContext.enteFile.id)
+                    fileContext.enteFile
                 );
             }
 

apps/photos/src/services/migrateThumbnailService.ts

@@ -1,4 +1,4 @@
-import downloadManager from 'services/downloadManager';
+import downloadManager from 'services/download';
 import { getLocalFiles } from 'services/fileService';
 import { generateThumbnail } from 'services/upload/thumbnailService';
 import { getToken } from '@ente/shared/storage/localStorage/helpers';
@@ -44,7 +44,6 @@ export async function replaceThumbnail(
 ) {
     let completedWithError = false;
     try {
-        const token = getToken();
         const cryptoWorker = await ComlinkCryptoWorker.getInstance();
         const files = await getLocalFiles();
         const trashFiles = await getLocalTrashedFiles();
@@ -69,8 +68,7 @@ export async function replaceThumbnail(
                     current: idx,
                     total: largeThumbnailFiles.length,
                 });
-                const originalThumbnail = await downloadManager.downloadThumb(
-                    token,
+                const originalThumbnail = await downloadManager.getThumbnail(
                     file
                 );
                 const dummyImageFile = new File(

apps/photos/src/services/publicCollectionDownloadManager.ts

@@ -1,314 +0,0 @@
-import {
-    getPublicCollectionFileURL,
-    getPublicCollectionThumbnailURL,
-} from '@ente/shared/network/api';
-import {
-    generateStreamFromArrayBuffer,
-    getRenderableFileURL,
-    createTypedObjectURL,
-} from 'utils/file';
-import HTTPService from '@ente/shared/network/HTTPService';
-import { EnteFile } from 'types/file';
-
-import { logError } from '@ente/shared/sentry';
-import { FILE_TYPE } from 'constants/file';
-import { CustomError } from '@ente/shared/error';
-import ComlinkCryptoWorker from '@ente/shared/crypto';
-import { CACHES } from '@ente/shared/storage/cacheStorage/constants';
-import { CacheStorageService } from '@ente/shared/storage/cacheStorage';
-import { LimitedCache } from '@ente/shared/storage/cacheStorage/types';
-
-class PublicCollectionDownloadManager {
-    private fileObjectURLPromise = new Map<
-        string,
-        Promise<{ original: string[]; converted: string[] }>
-    >();
-    private thumbnailObjectURLPromise = new Map<number, Promise<string>>();
-
-    private fileDownloadProgress = new Map<number, number>();
-
-    private progressUpdater: (value: Map<number, number>) => void;
-
-    setProgressUpdater(progressUpdater: (value: Map<number, number>) => void) {
-        this.progressUpdater = progressUpdater;
-    }
-
-    private async getThumbnailCache() {
-        try {
-            const thumbnailCache = await CacheStorageService.open(
-                CACHES.THUMBS
-            );
-            return thumbnailCache;
-        } catch (e) {
-            return null;
-            // ignore
-        }
-    }
-
-    public async getCachedThumbnail(
-        file: EnteFile,
-        thumbnailCache?: LimitedCache
-    ) {
-        try {
-            if (!thumbnailCache) {
-                thumbnailCache = await this.getThumbnailCache();
-            }
-            const cacheResp: Response = await thumbnailCache?.match(
-                file.id.toString()
-            );
-
-            if (cacheResp) {
-                return URL.createObjectURL(await cacheResp.blob());
-            }
-            return null;
-        } catch (e) {
-            logError(e, 'failed to get cached thumbnail');
-            throw e;
-        }
-    }
-
-    public async getThumbnail(
-        file: EnteFile,
-        token: string,
-        passwordToken: string
-    ) {
-        try {
-            if (!token) {
-                return null;
-            }
-
-            if (!this.thumbnailObjectURLPromise.has(file.id)) {
-                const downloadPromise = async () => {
-                    const thumbnailCache = await this.getThumbnailCache();
-                    const cachedThumb = await this.getCachedThumbnail(
-                        file,
-                        thumbnailCache
-                    );
-                    if (cachedThumb) {
-                        return cachedThumb;
-                    }
-
-                    const thumb = await this.downloadThumb(
-                        token,
-                        passwordToken,
-                        file
-                    );
-                    const thumbBlob = new Blob([thumb]);
-                    try {
-                        await thumbnailCache?.put(
-                            file.id.toString(),
-                            new Response(thumbBlob)
-                        );
-                    } catch (e) {
-                        // TODO: handle storage full exception.
-                    }
-                    return URL.createObjectURL(thumbBlob);
-                };
-                this.thumbnailObjectURLPromise.set(file.id, downloadPromise());
-            }
-
-            return await this.thumbnailObjectURLPromise.get(file.id);
-        } catch (e) {
-            this.thumbnailObjectURLPromise.delete(file.id);
-            logError(e, 'get publicDownloadManger preview Failed');
-            throw e;
-        }
-    }
-
-    private downloadThumb = async (
-        token: string,
-        passwordToken: string,
-        file: EnteFile
-    ) => {
-        const resp = await HTTPService.get(
-            getPublicCollectionThumbnailURL(file.id),
-            null,
-            {
-                'X-Auth-Access-Token': token,
-                ...(passwordToken && {
-                    'X-Auth-Access-Token-JWT': passwordToken,
-                }),
-            },
-            { responseType: 'arraybuffer' }
-        );
-        if (typeof resp.data === 'undefined') {
-            throw Error(CustomError.REQUEST_FAILED);
-        }
-        const cryptoWorker = await ComlinkCryptoWorker.getInstance();
-        const decrypted = await cryptoWorker.decryptThumbnail(
-            new Uint8Array(resp.data),
-            await cryptoWorker.fromB64(file.thumbnail.decryptionHeader),
-            file.key
-        );
-        return decrypted;
-    };
-
-    getFile = async (
-        file: EnteFile,
-        token: string,
-        passwordToken: string,
-        forPreview = false
-    ) => {
-        const fileKey = forPreview ? `${file.id}_preview` : `${file.id}`;
-        try {
-            const getFilePromise = async () => {
-                const fileStream = await this.downloadFile(
-                    token,
-                    passwordToken,
-                    file
-                );
-                const fileBlob = await new Response(fileStream).blob();
-                if (forPreview) {
-                    return await getRenderableFileURL(file, fileBlob);
-                } else {
-                    const fileURL = await createTypedObjectURL(
-                        fileBlob,
-                        file.metadata.title
-                    );
-                    return { converted: [fileURL], original: [fileURL] };
-                }
-            };
-
-            if (!this.fileObjectURLPromise.get(fileKey)) {
-                this.fileObjectURLPromise.set(fileKey, getFilePromise());
-            }
-            const fileURLs = await this.fileObjectURLPromise.get(fileKey);
-            return fileURLs;
-        } catch (e) {
-            this.fileObjectURLPromise.delete(fileKey);
-            logError(e, 'public download manager Failed to get File');
-            throw e;
-        }
-    };
-
-    public async getCachedOriginalFile(file: EnteFile) {
-        return await this.fileObjectURLPromise.get(file.id.toString());
-    }
-
-    async downloadFile(token: string, passwordToken: string, file: EnteFile) {
-        const cryptoWorker = await ComlinkCryptoWorker.getInstance();
-        if (!token) {
-            return null;
-        }
-        const onDownloadProgress = this.trackDownloadProgress(file.id);
-
-        if (
-            file.metadata.fileType === FILE_TYPE.IMAGE ||
-            file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
-        ) {
-            const resp = await HTTPService.get(
-                getPublicCollectionFileURL(file.id),
-                null,
-                {
-                    'X-Auth-Access-Token': token,
-                    ...(passwordToken && {
-                        'X-Auth-Access-Token-JWT': passwordToken,
-                        onDownloadProgress,
-                    }),
-                },
-                { responseType: 'arraybuffer' }
-            );
-            if (typeof resp.data === 'undefined') {
-                throw Error(CustomError.REQUEST_FAILED);
-            }
-            const decrypted = await cryptoWorker.decryptFile(
-                new Uint8Array(resp.data),
-                await cryptoWorker.fromB64(file.file.decryptionHeader),
-                file.key
-            );
-            return generateStreamFromArrayBuffer(decrypted);
-        }
-        const resp = await fetch(getPublicCollectionFileURL(file.id), {
-            headers: {
-                'X-Auth-Access-Token': token,
-                ...(passwordToken && {
-                    'X-Auth-Access-Token-JWT': passwordToken,
-                }),
-            },
-        });
-        const reader = resp.body.getReader();
-
-        const contentLength = +resp.headers.get('Content-Length');
-        let downloadedBytes = 0;
-
-        const stream = new ReadableStream({
-            async start(controller) {
-                const decryptionHeader = await cryptoWorker.fromB64(
-                    file.file.decryptionHeader
-                );
-                const fileKey = await cryptoWorker.fromB64(file.key);
-                const { pullState, decryptionChunkSize } =
-                    await cryptoWorker.initChunkDecryption(
-                        decryptionHeader,
-                        fileKey
-                    );
-                let data = new Uint8Array();
-                // The following function handles each data chunk
-                function push() {
-                    // "done" is a Boolean and value a "Uint8Array"
-                    reader.read().then(async ({ done, value }) => {
-                        // Is there more data to read?
-                        if (!done) {
-                            downloadedBytes += value.byteLength;
-                            onDownloadProgress({
-                                loaded: downloadedBytes,
-                                total: contentLength,
-                            });
-                            const buffer = new Uint8Array(
-                                data.byteLength + value.byteLength
-                            );
-                            buffer.set(new Uint8Array(data), 0);
-                            buffer.set(new Uint8Array(value), data.byteLength);
-                            if (buffer.length > decryptionChunkSize) {
-                                const fileData = buffer.slice(
-                                    0,
-                                    decryptionChunkSize
-                                );
-                                const { decryptedData } =
-                                    await cryptoWorker.decryptFileChunk(
-                                        fileData,
-                                        pullState
-                                    );
-                                controller.enqueue(decryptedData);
-                                data = buffer.slice(decryptionChunkSize);
-                            } else {
-                                data = buffer;
-                            }
-                            push();
-                        } else {
-                            if (data) {
-                                const { decryptedData } =
-                                    await cryptoWorker.decryptFileChunk(
-                                        data,
-                                        pullState
-                                    );
-                                controller.enqueue(decryptedData);
-                                data = null;
-                            }
-                            controller.close();
-                        }
-                    });
-                }
-
-                push();
-            },
-        });
-        return stream;
-    }
-
-    trackDownloadProgress = (fileID: number) => {
-        return (event: { loaded: number; total: number }) => {
-            if (event.loaded === event.total) {
-                this.fileDownloadProgress.delete(fileID);
-            } else {
-                this.fileDownloadProgress.set(
-                    fileID,
-                    Math.round((event.loaded * 100) / event.total)
-                );
-            }
-            this.progressUpdater(new Map(this.fileDownloadProgress));
-        };
-    };
-}
-
-export default new PublicCollectionDownloadManager();

apps/photos/src/services/searchService.ts

@@ -32,6 +32,7 @@ import {
     computeClipMatchScore,
     getLocalClipImageEmbeddings,
 } from './clipService';
+import { CustomError } from '@ente/shared/error';
 
 const DIGITS = new Set(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']);
 
@@ -302,7 +303,9 @@ async function getClipSuggestion(searchPhrase: string): Promise<Suggestion> {
             label: searchPhrase,
         };
     } catch (e) {
-        logError(e, 'getClipSuggestion failed');
+        if (!e.message?.includes(CustomError.MODEL_DOWNLOAD_PENDING)) {
+            logError(e, 'getClipSuggestion failed');
+        }
         return null;
     }
 }

apps/photos/src/services/updateCreationTimeWithExif.ts

@@ -2,11 +2,10 @@ import { FIX_OPTIONS } from 'components/FixCreationTime';
 import { SetProgressTracker } from 'components/FixLargeThumbnail';
 import {
     changeFileCreationTime,
-    getFileFromURL,
     updateExistingFilePubMetadata,
 } from 'utils/file';
 import { logError } from '@ente/shared/sentry';
-import downloadManager from './downloadManager';
+import downloadManager from './download';
 import { EnteFile } from 'types/file';
 
 import { getParsedExifData } from './upload/exifService';
@@ -43,9 +42,12 @@ export async function updateCreationTimeWithExif(
                     if (file.metadata.fileType !== FILE_TYPE.IMAGE) {
                         continue;
                     }
-                    const fileURL = (await downloadManager.getFile(file))
-                        .original[0];
-                    const fileObject = await getFileFromURL(fileURL);
+                    const fileStream = await downloadManager.getFile(file);
+                    const fileBlob = await new Response(fileStream).blob();
+                    const fileObject = new File(
+                        [fileBlob],
+                        file.metadata.title
+                    );
                     const fileTypeInfo = await getFileType(fileObject);
                     const exifData = await getParsedExifData(
                         fileObject,

apps/photos/src/services/watchFolder/watchFolderService.ts

@@ -9,7 +9,7 @@ import {
     WatchMapping,
     WatchMappingSyncedFile,
 } from 'types/watchFolder';
-import debounce from 'debounce-promise';
+import debounce from 'debounce';
 import {
     diskFileAddedCallback,
     diskFileRemovedCallback,
@@ -37,6 +37,11 @@ class watchFolderService {
     private setCollectionName: (collectionName: string) => void;
     private syncWithRemote: () => void;
     private setWatchFolderServiceIsRunning: (isRunning: boolean) => void;
+    private debouncedRunNextEvent: () => void;
+
+    constructor() {
+        this.debouncedRunNextEvent = debounce(() => this.runNextEvent(), 1000);
+    }
 
     isUploadRunning() {
         return this.uploadRunning;
@@ -160,7 +165,7 @@ class watchFolderService {
 
     pushEvent(event: EventQueueItem) {
         this.eventQueue.push(event);
-        debounce(this.runNextEvent.bind(this), 300)();
+        this.debouncedRunNextEvent();
     }
 
     async pushTrashedDir(path: string) {
@@ -255,7 +260,7 @@ class watchFolderService {
             } else {
                 await this.processTrashEvent();
                 this.setIsEventRunning(false);
-                this.runNextEvent();
+                setTimeout(() => this.runNextEvent(), 0);
             }
         } catch (e) {
             logError(e, 'runNextEvent failed');

apps/photos/src/types/deduplicate/index.ts

@@ -1,7 +1,4 @@
 export type DeduplicateContextType = {
-    clubSameTimeFilesOnly: boolean;
-    setClubSameTimeFilesOnly: (clubSameTimeFilesOnly: boolean) => void;
-    fileSizeMap: Map<number, number>;
     isOnDeduplicatePage: boolean;
     collectionNameMap: Map<number, string>;
 };

apps/photos/src/types/file/index.ts

@@ -1,3 +1,4 @@
+import { SourceURLs } from 'services/download';
 import {
     EncryptedMagicMetadata,
     MagicMetadataCore,
@@ -50,6 +51,7 @@ export interface EnteFile
     isTrashed?: boolean;
     key: string;
     src?: string;
+    srcURLs?: SourceURLs;
     msrc?: string;
     html?: string;
     w?: number;
@@ -57,9 +59,6 @@ export interface EnteFile
     title?: string;
     deleteBy?: number;
     isSourceLoaded?: boolean;
-    originalVideoURL?: string;
-    originalImageURL?: string;
-    dataIndex?: number;
     conversionFailed?: boolean;
     isConverted?: boolean;
 }

apps/photos/src/types/gallery/index.ts

@@ -31,8 +31,6 @@ export enum UploadTypeSelectorIntent {
     collectPhotos,
 }
 export type GalleryContextType = {
-    thumbs: Map<number, string>;
-    files: Map<number, MergedSourceURL>;
     showPlanSelectorModal: () => void;
     setActiveCollectionID: (collectionID: number) => void;
     syncWithRemote: (force?: boolean, silent?: boolean) => Promise<void>;

apps/photos/src/types/publicCollection/index.ts

@@ -2,7 +2,6 @@ import { TimeStampListItem } from 'components/PhotoList';
 import { REPORT_REASON } from 'constants/publicCollection';
 import { PublicURL } from 'types/collection';
 import { EnteFile } from 'types/file';
-import { MergedSourceURL } from 'types/gallery';
 
 export interface PublicCollectionGalleryContextType {
     token: string;
@@ -11,8 +10,6 @@ export interface PublicCollectionGalleryContextType {
     accessedThroughSharedURL: boolean;
     photoListHeader: TimeStampListItem;
     photoListFooter: TimeStampListItem;
-    thumbs: Map<number, string>;
-    files: Map<number, MergedSourceURL>;
 }
 
 export interface LocalSavedPublicCollectionFiles {

apps/photos/src/utils/billing/index.ts

@@ -140,11 +140,14 @@ export function hasMobileSubscription(subscription: Subscription) {
 }
 
 export function hasExceededStorageQuota(userDetails: UserDetails) {
+    const bonusStorage = userDetails.storageBonus ?? 0;
     if (isPartOfFamily(userDetails.familyData)) {
         const usage = getTotalFamilyUsage(userDetails.familyData);
-        return usage > userDetails.familyData.storage;
+        return usage > userDetails.familyData.storage + bonusStorage;
     } else {
-        return userDetails.usage > userDetails.subscription.storage;
+        return (
+            userDetails.usage > userDetails.subscription.storage + bonusStorage
+        );
     }
 }
 

apps/photos/src/utils/file/index.ts

@@ -10,7 +10,10 @@ import {
 } from 'types/file';
 import { decodeLivePhoto } from 'services/livePhotoService';
 import { getFileType } from 'services/typeDetectionService';
-import DownloadManager from 'services/downloadManager';
+import DownloadManager, {
+    LivePhotoSourceURL,
+    SourceURLs,
+} from 'services/download';
 import { logError } from '@ente/shared/sentry';
 import { User } from '@ente/shared/user/types';
 import { getData, LS_KEYS } from '@ente/shared/storage/localStorage';
@@ -24,7 +27,6 @@ import {
     SUPPORTED_RAW_FORMATS,
     RAW_FORMATS,
 } from 'constants/file';
-import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
 import heicConversionService from 'services/heicConversionService';
 import * as ffmpegService from 'services/ffmpeg/ffmpegService';
 import { VISIBILITY_STATE } from 'types/magicMetadata';
@@ -86,44 +88,12 @@ export async function getUpdatedEXIFFileForDownload(
     }
 }
 
-export async function downloadFile(
-    file: EnteFile,
-    accessedThroughSharedURL: boolean,
-    token?: string,
-    passwordToken?: string
-) {
+export async function downloadFile(file: EnteFile) {
     try {
-        let fileBlob: Blob;
         const fileReader = new FileReader();
-        if (accessedThroughSharedURL) {
-            const fileURL =
-                await PublicCollectionDownloadManager.getCachedOriginalFile(
-                    file
-                )[0];
-            if (!fileURL) {
-                fileBlob = await new Response(
-                    await PublicCollectionDownloadManager.downloadFile(
-                        token,
-                        passwordToken,
-                        file
-                    )
-                ).blob();
-            } else {
-                fileBlob = await (await fetch(fileURL)).blob();
-            }
-        } else {
-            const fileURL = await DownloadManager.getCachedOriginalFile(
-                file
-            )[0];
-            if (!fileURL) {
-                fileBlob = await new Response(
-                    await DownloadManager.downloadFile(file)
-                ).blob();
-            } else {
-                fileBlob = await (await fetch(fileURL)).blob();
-            }
-        }
-
+        let fileBlob = await new Response(
+            await DownloadManager.getFile(file)
+        ).blob();
         if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
             const livePhoto = await decodeLivePhoto(file, fileBlob);
             const image = new File([livePhoto.image], livePhoto.imageNameTitle);
@@ -310,83 +280,124 @@ export function generateStreamFromArrayBuffer(data: Uint8Array) {
     });
 }
 
-export async function getRenderableFileURL(file: EnteFile, fileBlob: Blob) {
+export async function getRenderableFileURL(
+    file: EnteFile,
+    fileBlob: Blob,
+    originalFileURL: string,
+    forceConvert: boolean
+): Promise<SourceURLs> {
+    let srcURLs: SourceURLs['url'];
     switch (file.metadata.fileType) {
         case FILE_TYPE.IMAGE: {
             const convertedBlob = await getRenderableImage(
                 file.metadata.title,
                 fileBlob
             );
-            const { originalURL, convertedURL } = getFileObjectURLs(
+            const convertedURL = getFileObjectURL(
+                originalFileURL,
                 fileBlob,
                 convertedBlob
             );
-            return {
-                converted: [convertedURL],
-                original: [originalURL],
-            };
+            srcURLs = convertedURL;
+            break;
         }
         case FILE_TYPE.LIVE_PHOTO: {
-            return await getRenderableLivePhotoURL(file, fileBlob);
+            srcURLs = await getRenderableLivePhotoURL(
+                file,
+                fileBlob,
+                forceConvert
+            );
+            break;
         }
         case FILE_TYPE.VIDEO: {
             const convertedBlob = await getPlayableVideo(
                 file.metadata.title,
-                fileBlob
+                fileBlob,
+                forceConvert
             );
-            const { originalURL, convertedURL } = getFileObjectURLs(
+            const convertedURL = getFileObjectURL(
+                originalFileURL,
                 fileBlob,
                 convertedBlob
             );
-            return {
-                converted: [convertedURL],
-                original: [originalURL],
-            };
+            srcURLs = convertedURL;
+            break;
         }
         default: {
-            const previewURL = await createTypedObjectURL(
-                fileBlob,
-                file.metadata.title
-            );
-            return {
-                converted: [previewURL],
-                original: [previewURL],
-            };
+            srcURLs = originalFileURL;
+            break;
         }
     }
+
+    let isOriginal: boolean;
+    if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
+        isOriginal = false;
+    } else {
+        isOriginal = (srcURLs as string) === (originalFileURL as string);
+    }
+
+    return {
+        url: srcURLs,
+        isOriginal,
+        isRenderable:
+            file.metadata.fileType !== FILE_TYPE.LIVE_PHOTO && !!srcURLs,
+        type:
+            file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
+                ? 'livePhoto'
+                : 'normal',
+    };
 }
 
 async function getRenderableLivePhotoURL(
     file: EnteFile,
-    fileBlob: Blob
-): Promise<{ original: string[]; converted: string[] }> {
+    fileBlob: Blob,
+    forceConvert: boolean
+): Promise<LivePhotoSourceURL> {
     const livePhoto = await decodeLivePhoto(file, fileBlob);
-    const imageBlob = new Blob([livePhoto.image]);
-    const videoBlob = new Blob([livePhoto.video]);
-    const convertedImageBlob = await getRenderableImage(
-        livePhoto.imageNameTitle,
-        imageBlob
-    );
-    const convertedVideoBlob = await getPlayableVideo(
-        livePhoto.videoNameTitle,
-        videoBlob,
-        true
-    );
-    const { originalURL: originalImageURL, convertedURL: convertedImageURL } =
-        getFileObjectURLs(imageBlob, convertedImageBlob);
 
-    const { originalURL: originalVideoURL, convertedURL: convertedVideoURL } =
-        getFileObjectURLs(videoBlob, convertedVideoBlob);
+    const getRenderableLivePhotoImageURL = async () => {
+        try {
+            const imageBlob = new Blob([livePhoto.image]);
+            const convertedImageBlob = await getRenderableImage(
+                livePhoto.imageNameTitle,
+                imageBlob
+            );
+
+            return URL.createObjectURL(convertedImageBlob);
+        } catch (e) {
+            //ignore and return null
+            return null;
+        }
+    };
+
+    const getRenderableLivePhotoVideoURL = async () => {
+        try {
+            const videoBlob = new Blob([livePhoto.video]);
+
+            const convertedVideoBlob = await getPlayableVideo(
+                livePhoto.videoNameTitle,
+                videoBlob,
+                forceConvert,
+                true
+            );
+            return URL.createObjectURL(convertedVideoBlob);
+        } catch (e) {
+            //ignore and return null
+            return null;
+        }
+    };
+
     return {
-        converted: [convertedImageURL, convertedVideoURL],
-        original: [originalImageURL, originalVideoURL],
+        image: getRenderableLivePhotoImageURL,
+        video: getRenderableLivePhotoVideoURL,
     };
 }
 
 export async function getPlayableVideo(
     videoNameTitle: string,
     videoBlob: Blob,
-    forceConvert = false
+    forceConvert = false,
+    runOnWeb = false
 ) {
     try {
         const isPlayable = await isPlaybackPossible(
@@ -395,7 +406,7 @@ export async function getPlayableVideo(
         if (isPlayable && !forceConvert) {
             return videoBlob;
         } else {
-            if (!forceConvert && !isElectron()) {
+            if (!forceConvert && !runOnWeb && !isElectron()) {
                 return null;
             }
             addLogLine(
@@ -594,9 +605,9 @@ export function updateExistingFilePubMetadata(
     existingFile.metadata = mergeMetadata([existingFile])[0].metadata;
 }
 
-export async function getFileFromURL(fileURL: string) {
+export async function getFileFromURL(fileURL: string, name: string) {
     const fileBlob = await (await fetch(fileURL)).blob();
-    const fileFile = new File([fileBlob], 'temp');
+    const fileFile = new File([fileBlob], name);
     return fileFile;
 }
 
@@ -627,7 +638,7 @@ export async function downloadFiles(
             if (progressBarUpdater?.isCancelled()) {
                 return;
             }
-            await downloadFile(file, false);
+            await downloadFile(file);
             progressBarUpdater?.increaseSuccess();
         } catch (e) {
             logError(e, 'download fail for file');
@@ -665,13 +676,9 @@ export async function downloadFileDesktop(
     file: EnteFile,
     downloadPath: string
 ) {
-    let fileStream: ReadableStream<Uint8Array>;
-    const fileURL = await DownloadManager.getCachedOriginalFile(file)[0];
-    if (!fileURL) {
-        fileStream = await DownloadManager.downloadFile(file);
-    } else {
-        fileStream = await fetch(fileURL).then((res) => res.body);
-    }
+    const fileStream = (await DownloadManager.getFile(
+        file
+    )) as ReadableStream<Uint8Array>;
     const updatedFileStream = await getUpdatedEXIFFileForDownload(
         fileReader,
         file,
@@ -939,12 +946,15 @@ const fixTimeHelper = async (
     setFixCreationTimeAttributes({ files: selectedFiles });
 };
 
-const getFileObjectURLs = (originalBlob: Blob, convertedBlob: Blob) => {
-    const originalURL = URL.createObjectURL(originalBlob);
+const getFileObjectURL = (
+    originalFileURL: string,
+    originalBlob: Blob,
+    convertedBlob: Blob
+) => {
     const convertedURL = convertedBlob
         ? convertedBlob === originalBlob
-            ? originalURL
+            ? originalFileURL
             : URL.createObjectURL(convertedBlob)
         : null;
-    return { originalURL, convertedURL };
+    return convertedURL;
 };

apps/photos/src/utils/machineLearning/index.ts

@@ -1,12 +1,9 @@
 import { NormalizedFace } from 'blazeface-back';
 import * as tf from '@tensorflow/tfjs-core';
-import {
-    BLAZEFACE_FACE_SIZE,
-    ML_SYNC_DOWNLOAD_TIMEOUT_MS,
-} from 'constants/mlConfig';
+import { BLAZEFACE_FACE_SIZE } from 'constants/mlConfig';
 import { euclidean } from 'hdbscan';
 import PQueue from 'p-queue';
-import DownloadManager from 'services/downloadManager';
+import DownloadManager from 'services/download';
 import { getLocalFiles } from 'services/fileService';
 import { EnteFile } from 'types/file';
 import { Dimensions } from 'types/image';
@@ -40,8 +37,6 @@ import { CACHES } from '@ente/shared/storage/cacheStorage/constants';
 import { FILE_TYPE } from 'constants/file';
 import { decodeLivePhoto } from 'services/livePhotoService';
 import { addLogLine } from '@ente/shared/logging';
-import { Remote } from 'comlink';
-import { DedicatedCryptoWorker } from '@ente/shared/crypto/internal/crypto.worker';
 
 export function f32Average(descriptors: Float32Array[]) {
     if (descriptors.length < 1) {
@@ -229,7 +224,7 @@ export async function getFaceImage(
         file = await getLocalFile(face.fileId);
     }
 
-    const imageBitmap = await getOriginalImageBitmap(file, token);
+    const imageBitmap = await getOriginalImageBitmap(file);
     const faceImageBitmap = ibExtractFaceImage(
         imageBitmap,
         face.alignment,
@@ -325,39 +320,18 @@ export async function getImageBlobBitmap(blob: Blob): Promise<ImageBitmap> {
 //     return new TFImageBitmap(undefined, tfImage);
 // }
 
-async function getOriginalFile(
-    file: EnteFile,
-    token: string,
-    enteWorker?: Remote<DedicatedCryptoWorker>,
-    queue?: PQueue
-) {
+async function getOriginalFile(file: EnteFile, queue?: PQueue) {
     let fileStream;
     if (queue) {
-        fileStream = await queue.add(() =>
-            DownloadManager.downloadFile(
-                file,
-                token,
-                enteWorker,
-                ML_SYNC_DOWNLOAD_TIMEOUT_MS
-            )
-        );
+        fileStream = await queue.add(() => DownloadManager.getFile(file));
     } else {
-        fileStream = await DownloadManager.downloadFile(
-            file,
-            token,
-            enteWorker
-        );
+        fileStream = await DownloadManager.getFile(file);
     }
     return new Response(fileStream).blob();
 }
 
-async function getOriginalConvertedFile(
-    file: EnteFile,
-    token: string,
-    enteWorker?: Remote<DedicatedCryptoWorker>,
-    queue?: PQueue
-) {
-    const fileBlob = await getOriginalFile(file, token, enteWorker, queue);
+async function getOriginalConvertedFile(file: EnteFile, queue?: PQueue) {
+    const fileBlob = await getOriginalFile(file, queue);
     if (file.metadata.fileType === FILE_TYPE.IMAGE) {
         return await getRenderableImage(file.metadata.title, fileBlob);
     } else {
@@ -371,8 +345,6 @@ async function getOriginalConvertedFile(
 
 export async function getOriginalImageBitmap(
     file: EnteFile,
-    token: string,
-    enteWorker?: Remote<DedicatedCryptoWorker>,
     queue?: PQueue,
     useCache: boolean = false
 ) {
@@ -380,37 +352,21 @@ export async function getOriginalImageBitmap(
 
     if (useCache) {
         fileBlob = await cached(CACHES.FILES, file.id.toString(), () => {
-            return getOriginalConvertedFile(file, token, enteWorker, queue);
+            return getOriginalConvertedFile(file, queue);
         });
     } else {
-        fileBlob = await getOriginalConvertedFile(
-            file,
-            token,
-            enteWorker,
-            queue
-        );
+        fileBlob = await getOriginalConvertedFile(file, queue);
     }
     addLogLine('[MLService] Got file: ', file.id.toString());
 
     return getImageBlobBitmap(fileBlob);
 }
 
-export async function getThumbnailImageBitmap(
-    file: EnteFile,
-    token: string,
-    enteWorker?: Remote<DedicatedCryptoWorker>
-) {
-    const fileUrl = await DownloadManager.getThumbnail(
-        file,
-        token,
-        enteWorker,
-        ML_SYNC_DOWNLOAD_TIMEOUT_MS
-    );
+export async function getThumbnailImageBitmap(file: EnteFile) {
+    const thumb = await DownloadManager.getThumbnail(file);
     addLogLine('[MLService] Got thumbnail: ', file.id.toString());
 
-    const thumbFile = await fetch(fileUrl);
-
-    return getImageBlobBitmap(await thumbFile.blob());
+    return getImageBlobBitmap(new Blob([thumb]));
 }
 
 export async function getLocalFileImageBitmap(

apps/photos/src/utils/photoFrame/index.ts

@@ -1,7 +1,7 @@
 import { FILE_TYPE } from 'constants/file';
 import { EnteFile } from 'types/file';
-import { MergedSourceURL } from 'types/gallery';
 import { logError } from '@ente/shared/sentry';
+import { LivePhotoSourceURL, SourceURLs } from 'services/download';
 
 const WAIT_FOR_VIDEO_PLAYBACK = 1 * 1000;
 
@@ -52,6 +52,8 @@ export async function pauseVideo(livePhotoVideo, livePhotoImage) {
 }
 
 export function updateFileMsrcProps(file: EnteFile, url: string) {
+    file.w = window.innerWidth;
+    file.h = window.innerHeight;
     file.msrc = url;
     file.isSourceLoaded = false;
     file.conversionFailed = false;
@@ -67,82 +69,57 @@ export function updateFileMsrcProps(file: EnteFile, url: string) {
     }
 }
 
-export async function updateFileSrcProps(
-    file: EnteFile,
-    mergedURL: MergedSourceURL
-) {
-    const urls = {
-        original: mergedURL.original.split(','),
-        converted: mergedURL.converted.split(','),
-    };
-    let originalImageURL;
-    let originalVideoURL;
-    let convertedImageURL;
-    let convertedVideoURL;
-    let originalURL;
-    let isConverted;
-    let conversionFailed;
-    if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
-        [originalImageURL, originalVideoURL] = urls.original;
-        [convertedImageURL, convertedVideoURL] = urls.converted;
-        isConverted =
-            originalVideoURL !== convertedVideoURL ||
-            originalImageURL !== convertedImageURL;
-        conversionFailed = !convertedVideoURL || !convertedImageURL;
-    } else if (file.metadata.fileType === FILE_TYPE.VIDEO) {
-        [originalVideoURL] = urls.original;
-        [convertedVideoURL] = urls.converted;
-        isConverted = originalVideoURL !== convertedVideoURL;
-        conversionFailed = !convertedVideoURL;
-    } else if (file.metadata.fileType === FILE_TYPE.IMAGE) {
-        [originalImageURL] = urls.original;
-        [convertedImageURL] = urls.converted;
-        isConverted = originalImageURL !== convertedImageURL;
-        conversionFailed = !convertedImageURL;
-    } else {
-        [originalURL] = urls.original;
-        isConverted = false;
-        conversionFailed = false;
-    }
-
-    const isPlayable = !isConverted || (isConverted && !conversionFailed);
-
+export async function updateFileSrcProps(file: EnteFile, srcURLs: SourceURLs) {
+    const { url, isRenderable, isOriginal } = srcURLs;
     file.w = window.innerWidth;
     file.h = window.innerHeight;
-    file.isSourceLoaded = true;
-    file.originalImageURL = originalImageURL;
-    file.originalVideoURL = originalVideoURL;
-    file.isConverted = isConverted;
-    file.conversionFailed = conversionFailed;
-
-    if (!isPlayable) {
+    file.isSourceLoaded =
+        file.metadata.fileType === FILE_TYPE.LIVE_PHOTO
+            ? srcURLs.type === 'livePhoto'
+            : true;
+    file.isConverted = !isOriginal;
+    file.conversionFailed = !isRenderable;
+    file.srcURLs = srcURLs;
+    if (!isRenderable) {
+        file.isSourceLoaded = true;
         return;
     }
 
     if (file.metadata.fileType === FILE_TYPE.VIDEO) {
         file.html = `
                 <video controls onContextMenu="return false;">
-                    <source src="${convertedVideoURL}" />
+                    <source src="${url}" />
                     Your browser does not support the video tag.
                 </video>
                 `;
     } else if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
-        file.html = `
+        if (srcURLs.type === 'normal') {
+            file.html = `
                 <div class = 'pswp-item-container'>
-                    <img id = "live-photo-image-${file.id}" src="${convertedImageURL}" onContextMenu="return false;"/>
-                    <video id = "live-photo-video-${file.id}" loop muted onContextMenu="return false;">
-                        <source src="${convertedVideoURL}" />
-                        Your browser does not support the video tag.
-                    </video>
+                    <img id = "live-photo-image-${file.id}" src="${url}" onContextMenu="return false;"/>
                 </div>
                 `;
+        } else {
+            const { image: imageURL, video: videoURL } =
+                url as LivePhotoSourceURL;
+
+            file.html = `
+            <div class = 'pswp-item-container'>
+                <img id = "live-photo-image-${file.id}" src="${imageURL}" onContextMenu="return false;"/>
+                <video id = "live-photo-video-${file.id}" loop muted onContextMenu="return false;">
+                    <source src="${videoURL}" />
+                    Your browser does not support the video tag.
+                </video>
+            </div>
+            `;
+        }
     } else if (file.metadata.fileType === FILE_TYPE.IMAGE) {
-        file.src = convertedImageURL;
+        file.src = url as string;
     } else {
         logError(
             Error(`unknown file type - ${file.metadata.fileType}`),
             'Unknown file type'
         );
-        file.src = originalURL;
+        file.src = url as string;
     }
 }

apps/photos/src/utils/publicCollectionGallery/index.ts

@@ -9,8 +9,6 @@ const defaultPublicCollectionGalleryContext: PublicCollectionGalleryContextType
         accessedThroughSharedURL: false,
         photoListHeader: null,
         photoListFooter: null,
-        files: new Map(),
-        thumbs: new Map(),
     };
 
 export const PublicCollectionGalleryContext =

apps/photos/src/utils/storage/mlIDbStorage.ts

@@ -263,6 +263,12 @@ class MLIDbStorage {
         await Promise.all(fileIds.map((fileId) => tx.store.delete(fileId)));
     }
 
+    public async getFace(fileID: number, faceId: string) {
+        const file = await this.getFile(fileID);
+        const face = file.faces.filter((f) => f.id === faceId);
+        return face[0];
+    }
+
     public async getAllFacesMap() {
         const startTime = Date.now();
         const db = await this.db;

apps/photos/src/worker/ml.worker.ts

@@ -37,6 +37,14 @@ export class DedicatedMLWorker implements MachineLearningWorker {
         return mlService.sync(token, userID);
     }
 
+    public async regenerateFaceCrop(
+        token: string,
+        userID: number,
+        faceID: string
+    ) {
+        return mlService.regenerateFaceCrop(token, userID, faceID);
+    }
+
     public close() {
         self.close();
     }

apps/photos/tests/upload.test.ts

@@ -61,6 +61,13 @@ const DATE_TIME_PARSING_TEST_FILE_NAMES = [
     },
 ];
 
+const DATE_TIME_PARSING_TEST_FILE_NAMES_MUST_FAIL = [
+    'Snapchat-431959199.mp4.',
+    'Snapchat-400000000.mp4',
+    'Snapchat-900000000.mp4',
+    'Snapchat-100-10-20-19-15-12',
+];
+
 const FILE_NAME_TO_JSON_NAME = [
     {
         filename: 'IMG20210211125718-edited.jpg',
@@ -387,6 +394,16 @@ function parseDateTimeFromFileNameTest() {
             }
         }
     );
+    DATE_TIME_PARSING_TEST_FILE_NAMES_MUST_FAIL.forEach((fileName) => {
+        const dateTime = tryToParseDateTime(fileName);
+        if (dateTime) {
+            throw Error(
+                `parseDateTimeFromFileNameTest failed ❌ ,
+                for ${fileName}
+                expected: null got: ${dateTime}`
+            );
+        }
+    });
     console.log('parseDateTimeFromFileNameTest passed ✅');
 }
 

packages/accounts/api/user.ts

@@ -25,8 +25,14 @@ export const sendOtt = (appName: APPS, email: string) => {
     });
 };
 
-export const verifyOtt = (email: string, ott: string) =>
-    HTTPService.post(`${ENDPOINT}/users/verify-email`, { email, ott });
+export const verifyOtt = (email: string, ott: string, referral: string) => {
+    const cleanedReferral = `web:${referral?.trim() || ''}`;
+    return HTTPService.post(`${ENDPOINT}/users/verify-email`, {
+        email,
+        ott,
+        source: cleanedReferral,
+    });
+};
 
 export const putAttributes = (token: string, keyAttributes: KeyAttributes) =>
     HTTPService.put(

packages/accounts/components/SignUp.tsx

@@ -11,7 +11,10 @@ import {
 import { isWeakPassword } from '@ente/accounts/utils';
 import { generateKeyAndSRPAttributes } from '@ente/accounts/utils/srp';
 
-import { setJustSignedUp } from '@ente/shared/storage/localStorage/helpers';
+import {
+    setJustSignedUp,
+    setLocalReferralSource,
+} from '@ente/shared/storage/localStorage/helpers';
 import { SESSION_KEYS } from '@ente/shared/storage/sessionStorage';
 import { PAGES } from '@ente/accounts/constants/pages';
 import {
@@ -19,8 +22,11 @@ import {
     Checkbox,
     FormControlLabel,
     FormGroup,
+    IconButton,
+    InputAdornment,
     Link,
     TextField,
+    Tooltip,
     Typography,
 } from '@mui/material';
 import FormPaperTitle from '@ente/shared/components/Form/FormPaper/Title';
@@ -34,11 +40,13 @@ import ShowHidePassword from '@ente/shared/components/Form/ShowHidePassword';
 import { APPS } from '@ente/shared/apps/constants';
 import { NextRouter } from 'next/router';
 import { logError } from '@ente/shared/sentry';
+import InfoOutlined from '@mui/icons-material/InfoOutlined';
 
 interface FormValues {
     email: string;
     passphrase: string;
     confirm: string;
+    referral: string;
 }
 
 interface SignUpProps {
@@ -63,7 +71,7 @@ export default function SignUp({ router, appName, login }: SignUpProps) {
     };
 
     const registerUser = async (
-        { email, passphrase, confirm }: FormValues,
+        { email, passphrase, confirm, referral }: FormValues,
         { setFieldError }: FormikHelpers<FormValues>
     ) => {
         try {
@@ -74,6 +82,7 @@ export default function SignUp({ router, appName, login }: SignUpProps) {
             setLoading(true);
             try {
                 setData(LS_KEYS.USER, { email });
+                setLocalReferralSource(referral);
                 await sendOtt(appName, email);
             } catch (e) {
                 setFieldError('confirm', `${t('UNKNOWN_ERROR')} ${e.message}`);
@@ -115,6 +124,7 @@ export default function SignUp({ router, appName, login }: SignUpProps) {
                     email: '',
                     passphrase: '',
                     confirm: '',
+                    referral: '',
                 }}
                 validationSchema={Yup.object().shape({
                     email: Yup.string()
@@ -192,12 +202,47 @@ export default function SignUp({ router, appName, login }: SignUpProps) {
                             <PasswordStrengthHint
                                 password={values.passphrase}
                             />
+
+                            <Box sx={{ width: '100%' }}>
+                                <Typography
+                                    textAlign={'left'}
+                                    color="text.secondary"
+                                    mt={'24px'}>
+                                    {t('REFERRAL_CODE_HINT')}
+                                </Typography>
+                                <TextField
+                                    hiddenLabel
+                                    InputProps={{
+                                        endAdornment: (
+                                            <InputAdornment position="end">
+                                                <Tooltip
+                                                    title={t('REFERRAL_INFO')}>
+                                                    <IconButton
+                                                        tabIndex={-1}
+                                                        color="secondary"
+                                                        edge={'end'}>
+                                                        <InfoOutlined />
+                                                    </IconButton>
+                                                </Tooltip>
+                                            </InputAdornment>
+                                        ),
+                                    }}
+                                    fullWidth
+                                    name="referral"
+                                    type="text"
+                                    value={values.referral}
+                                    onChange={handleChange('referral')}
+                                    error={Boolean(errors.referral)}
+                                    disabled={loading}
+                                />
+                            </Box>
                             <FormGroup sx={{ width: '100%' }}>
                                 <FormControlLabel
                                     sx={{
                                         color: 'text.muted',
                                         ml: 0,
                                         mt: 2,
+                                        mb: 0,
                                     }}
                                     control={
                                         <Checkbox
@@ -234,7 +279,7 @@ export default function SignUp({ router, appName, login }: SignUpProps) {
                                 />
                             </FormGroup>
                         </VerticallyCentered>
-                        <Box my={4}>
+                        <Box mb={4}>
                             <SubmitButton
                                 sx={{ my: 0 }}
                                 buttonText={t('CREATE_ACCOUNT')}

packages/accounts/pages/credentials.tsx

@@ -73,7 +73,11 @@ export default function Credentials({
             setUser(user);
             let key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
             if (!key && isElectron()) {
-                key = await ElectronAPIs.getEncryptionKey();
+                try {
+                    key = await ElectronAPIs.getEncryptionKey();
+                } catch (e) {
+                    logError(e, 'getEncryptionKey failed');
+                }
                 if (key) {
                     await saveKeyInSessionStore(
                         SESSION_KEYS.ENCRYPTION_KEY,
@@ -220,7 +224,7 @@ export default function Credentials({
             }
             const redirectURL = InMemoryStore.get(MS_KEYS.REDIRECT_URL);
             InMemoryStore.delete(MS_KEYS.REDIRECT_URL);
-            router.push(APP_HOMES.get(redirectURL ?? appName));
+            router.push(redirectURL ?? APP_HOMES.get(appName));
         } catch (e) {
             logError(e, 'useMasterPassword failed');
         }

packages/accounts/pages/verify.tsx

@@ -7,7 +7,10 @@ import { verifyOtt, sendOtt, putAttributes } from '../api/user';
 import { logoutUser } from '../services/user';
 import { configureSRP } from '../services/srp';
 import { clearFiles } from '@ente/shared/storage/localForage/helpers';
-import { setIsFirstLogin } from '@ente/shared/storage/localStorage/helpers';
+import {
+    getLocalReferralSource,
+    setIsFirstLogin,
+} from '@ente/shared/storage/localStorage/helpers';
 import { clearKeys } from '@ente/shared/storage/sessionStorage';
 import { PAGES } from '../constants/pages';
 import { KeyAttributes, User } from '@ente/shared/user/types';
@@ -58,7 +61,8 @@ export default function VerifyPage({ appContext, router, appName }: PageProps) {
         setFieldError
     ) => {
         try {
-            const resp = await verifyOtt(email, ott);
+            const referralSource = getLocalReferralSource();
+            const resp = await verifyOtt(email, ott, referralSource);
             const {
                 keyAttributes,
                 encryptedToken,

packages/shared/apps/env.ts

@@ -1,7 +1,7 @@
 import { APP_ENV } from './constants';
 
 export const getAppEnv = () =>
-    process.env.NEXT_PUBLIC_APP_ENV ?? APP_ENV.DEVELOPMENT;
+    process.env.NEXT_PUBLIC_APP_ENV ?? APP_ENV.PRODUCTION;
 
 export const isDisableSentryFlagSet = () => {
     return process.env.NEXT_PUBLIC_DISABLE_SENTRY === 'true';

packages/shared/components/Navbar/SidebarToggler.tsx

@@ -6,7 +6,7 @@ interface Iprops {
 }
 export default function SidebarToggler({ openSidebar }: Iprops) {
     return (
-        <IconButton onClick={openSidebar}>
+        <IconButton onClick={openSidebar} sx={{ pl: 0 }}>
             <MenuIcon />
         </IconButton>
     );

packages/shared/components/Navbar/base.tsx

@@ -10,7 +10,9 @@ const NavbarBase = styled(FlexWrapper)<{ isMobile: boolean }>`
     background-color: ${({ theme }) => theme.colors.background.base};
     margin-bottom: 16px;
     padding: 0 24px;
-    padding: ${(props) => (props.isMobile ? '0 16px;' : '0 4px;')};
+    @media (max-width: 720px) {
+        padding: 0 4px;
+    }
 `;
 
 export default NavbarBase;

packages/shared/constants/pages.tsx

@@ -15,8 +15,6 @@ export enum PHOTOS_PAGES {
     SHARED_ALBUMS = '/shared-albums',
     // ML_DEBUG = '/ml-debug',
     DEDUPLICATE = '/deduplicate',
-    // AUTH page is used to show (auth)enticator codes
-    AUTH = '/auth',
 }
 
 export enum AUTH_PAGES {

packages/shared/crypto/helpers.ts

@@ -8,6 +8,7 @@ import { setRecoveryKey } from '@ente/accounts/api/user';
 import { logError } from '@ente/shared/sentry';
 import isElectron from 'is-electron';
 import ElectronAPIs from '../electron';
+import { addLogLine } from '../logging';
 
 const LOGIN_SUB_KEY_LENGTH = 32;
 const LOGIN_SUB_KEY_ID = 1;
@@ -104,7 +105,7 @@ export const saveKeyInSessionStore = async (
         key
     );
     setKey(keyType, sessionKeyAttributes);
-    console.log('fromDesktop', fromDesktop);
+    addLogLine('fromDesktop', fromDesktop);
     if (
         isElectron() &&
         !fromDesktop &&

packages/shared/electron/service.ts

@@ -0,0 +1,92 @@
+import * as Comlink from 'comlink';
+import { LimitedCache } from '@ente/shared/storage/cacheStorage/types';
+import {
+    ProxiedWorkerLimitedCache,
+    WorkerSafeElectronClient,
+} from './worker/client';
+import { wrap } from 'comlink';
+import { deserializeToResponse, serializeResponse } from './worker/utils/proxy';
+import { runningInWorker } from '@ente/shared/platform';
+import { ElectronAPIsType } from './types';
+
+export interface LimitedElectronAPIs
+    extends Pick<
+        ElectronAPIsType,
+        | 'openDiskCache'
+        | 'deleteDiskCache'
+        | 'getSentryUserID'
+        | 'convertToJPEG'
+        | 'logToDisk'
+    > {}
+
+class WorkerSafeElectronServiceImpl implements LimitedElectronAPIs {
+    proxiedElectron:
+        | Comlink.Remote<WorkerSafeElectronClient>
+        | WorkerSafeElectronClient;
+    ready: Promise<any>;
+
+    constructor() {
+        this.ready = this.init();
+    }
+    private async init() {
+        if (runningInWorker()) {
+            const workerSafeElectronClient =
+                wrap<typeof WorkerSafeElectronClient>(self);
+
+            this.proxiedElectron = await new workerSafeElectronClient();
+        } else {
+            this.proxiedElectron = new WorkerSafeElectronClient();
+        }
+    }
+    async openDiskCache(cacheName: string, cacheLimitInBytes?: number) {
+        await this.ready;
+        const cache = await this.proxiedElectron.openDiskCache(
+            cacheName,
+            cacheLimitInBytes
+        );
+        return {
+            match: transformMatch(cache.match.bind(cache)),
+            put: transformPut(cache.put.bind(cache)),
+            delete: cache.delete.bind(cache),
+        };
+    }
+
+    async deleteDiskCache(cacheName: string) {
+        await this.ready;
+        return await this.proxiedElectron.deleteDiskCache(cacheName);
+    }
+
+    async getSentryUserID() {
+        await this.ready;
+        return this.proxiedElectron.getSentryUserID();
+    }
+    async convertToJPEG(
+        inputFileData: Uint8Array,
+        filename: string
+    ): Promise<Uint8Array> {
+        await this.ready;
+        return this.proxiedElectron.convertToJPEG(inputFileData, filename);
+    }
+    async logToDisk(message: string) {
+        await this.ready;
+        return this.proxiedElectron.logToDisk(message);
+    }
+}
+
+export const WorkerSafeElectronService = new WorkerSafeElectronServiceImpl();
+
+function transformMatch(
+    fn: ProxiedWorkerLimitedCache['match']
+): LimitedCache['match'] {
+    return async (key: string, options) => {
+        return deserializeToResponse(await fn(key, options));
+    };
+}
+
+function transformPut(
+    fn: ProxiedWorkerLimitedCache['put']
+): LimitedCache['put'] {
+    return async (key: string, data: Response) => {
+        fn(key, await serializeResponse(data));
+    };
+}

packages/shared/electron/types.ts

@@ -59,7 +59,10 @@ export interface ElectronAPIsType {
     clearElectronStore: () => void;
     setEncryptionKey: (encryptionKey: string) => Promise<void>;
     getEncryptionKey: () => Promise<string>;
-    openDiskCache: (cacheName: string) => Promise<LimitedCache>;
+    openDiskCache: (
+        cacheName: string,
+        cacheLimitInBytes?: number
+    ) => Promise<LimitedCache>;
     deleteDiskCache: (cacheName: string) => Promise<boolean>;
     logToDisk: (msg: string) => void;
     convertToJPEG: (
@@ -97,4 +100,6 @@ export interface ElectronAPIsType {
     computeImageEmbedding: (imageData: Uint8Array) => Promise<Float32Array>;
     computeTextEmbedding: (text: string) => Promise<Float32Array>;
     getPlatform: () => Promise<'mac' | 'windows' | 'linux'>;
+    setCustomCacheDirectory: (directory: string) => Promise<void>;
+    getCacheDirectory: () => Promise<string>;
 }

packages/shared/electron/worker/client.ts

@@ -0,0 +1,74 @@
+import * as Comlink from 'comlink';
+import { LimitedCache } from '@ente/shared/storage/cacheStorage/types';
+import { serializeResponse, deserializeToResponse } from './utils/proxy';
+import ElectronAPIs from '@ente/shared/electron';
+
+export interface ProxiedLimitedElectronAPIs {
+    openDiskCache: (
+        cacheName: string,
+        cacheLimitInBytes?: number
+    ) => Promise<ProxiedWorkerLimitedCache>;
+    deleteDiskCache: (cacheName: string) => Promise<boolean>;
+    getSentryUserID: () => Promise<string>;
+    convertToJPEG: (
+        inputFileData: Uint8Array,
+        filename: string
+    ) => Promise<Uint8Array>;
+    logToDisk: (message: string) => void;
+}
+export interface ProxiedWorkerLimitedCache {
+    match: (
+        key: string,
+        options?: { sizeInBytes?: number }
+    ) => Promise<ArrayBuffer>;
+    put: (key: string, data: ArrayBuffer) => Promise<void>;
+    delete: (key: string) => Promise<boolean>;
+}
+
+export class WorkerSafeElectronClient implements ProxiedLimitedElectronAPIs {
+    async openDiskCache(cacheName: string, cacheLimitInBytes?: number) {
+        const cache = await ElectronAPIs.openDiskCache(
+            cacheName,
+            cacheLimitInBytes
+        );
+        return Comlink.proxy({
+            match: Comlink.proxy(transformMatch(cache.match.bind(cache))),
+            put: Comlink.proxy(transformPut(cache.put.bind(cache))),
+            delete: Comlink.proxy(cache.delete.bind(cache)),
+        });
+    }
+
+    async deleteDiskCache(cacheName: string) {
+        return await ElectronAPIs.deleteDiskCache(cacheName);
+    }
+
+    async getSentryUserID() {
+        return await ElectronAPIs.getSentryUserID();
+    }
+
+    async convertToJPEG(
+        inputFileData: Uint8Array,
+        filename: string
+    ): Promise<Uint8Array> {
+        return await ElectronAPIs.convertToJPEG(inputFileData, filename);
+    }
+    logToDisk(message: string) {
+        return ElectronAPIs.logToDisk(message);
+    }
+}
+
+function transformMatch(
+    fn: LimitedCache['match']
+): ProxiedWorkerLimitedCache['match'] {
+    return async (key: string, options: { sizeInBytes?: number }) => {
+        return serializeResponse(await fn(key, options));
+    };
+}
+
+function transformPut(
+    fn: LimitedCache['put']
+): ProxiedWorkerLimitedCache['put'] {
+    return async (key: string, data: ArrayBuffer) => {
+        fn(key, deserializeToResponse(data));
+    };
+}

packages/shared/error/index.ts

@@ -86,6 +86,12 @@ export const CustomError = {
     ServerError: 'server error',
     FILE_NOT_FOUND: 'file not found',
     UNSUPPORTED_PLATFORM: 'Unsupported platform',
+    MODEL_DOWNLOAD_PENDING:
+        'Model download pending, skipping clip search request',
+    DOWNLOAD_MANAGER_NOT_READY: 'Download manager not initialized',
+    UPDATE_URL_FILE_ID_MISMATCH: 'update url file id mismatch',
+    URL_ALREADY_SET: 'url already set',
+    FILE_CONVERSION_FAILED: 'file conversion failed',
 };
 
 export function handleUploadError(error: any): Error {

packages/shared/logging/index.ts

@@ -1,9 +1,9 @@
 import isElectron from 'is-electron';
-import ElectronAPIs from '@ente/shared/electron';
 import { logError } from '@ente/shared/sentry';
 import { getAppEnv } from '../apps/env';
 import { APP_ENV } from '../apps/constants';
 import { formatLog, logWeb } from './web';
+import { WorkerSafeElectronService } from '../electron/service';
 
 export const MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB
 export const MAX_LOG_LINES = 1000;
@@ -18,7 +18,7 @@ export function addLogLine(
             console.log(completeLog);
         }
         if (isElectron()) {
-            ElectronAPIs.logToDisk(completeLog);
+            WorkerSafeElectronService.logToDisk(completeLog);
         } else {
             logWeb(completeLog);
         }

packages/shared/next/env.js

@@ -9,7 +9,7 @@ module.exports = {
 };
 
 module.exports.getAppEnv = () => {
-    return process.env.NEXT_PUBLIC_APP_ENV ?? ENV_DEVELOPMENT;
+    return process.env.NEXT_PUBLIC_APP_ENV ?? ENV_PRODUCTION;
 };
 
 module.exports.isDisableSentryFlagSet = () => {

packages/shared/sentry/utils.ts

@@ -1,4 +1,4 @@
-import ElectronAPIs from '@ente/shared/electron';
+import { WorkerSafeElectronService } from '@ente/shared/electron/service';
 import {
     getLocalSentryUserID,
     setLocalSentryUserID,
@@ -12,7 +12,7 @@ import { HttpStatusCode } from 'axios';
 
 export async function getSentryUserID() {
     if (isElectron()) {
-        return await ElectronAPIs.getSentryUserID();
+        return await WorkerSafeElectronService.getSentryUserID();
     } else {
         let anonymizeUserID = getLocalSentryUserID();
         if (!anonymizeUserID) {