diff --git a/.yarn/install-state.gz b/.yarn/install-state.gz index 12b9562b5..5e15c15b5 100644 Binary files a/.yarn/install-state.gz and b/.yarn/install-state.gz differ diff --git a/apps/auth/src/components/OTPDisplay.tsx b/apps/auth/src/components/OTPDisplay.tsx index a10c78dc1..3d445a2fe 100644 --- a/apps/auth/src/components/OTPDisplay.tsx +++ b/apps/auth/src/components/OTPDisplay.tsx @@ -1,6 +1,6 @@ import React, { useState, useEffect } from 'react'; import { TOTP, HOTP } from 'otpauth'; -import { Code } from 'types/authenticator/code'; +import { Code } from 'types/code'; import TimerProgress from './TimerProgress'; import { t } from 'i18next'; import { ButtonBase, Snackbar } from '@mui/material'; diff --git a/apps/auth/src/pages/auth/index.tsx b/apps/auth/src/pages/auth/index.tsx index ec9ba6be5..451b6fb47 100644 --- a/apps/auth/src/pages/auth/index.tsx +++ b/apps/auth/src/pages/auth/index.tsx @@ -1,6 +1,6 @@ import React, { useContext, useEffect, useState } from 'react'; import OTPDisplay from 'components/OTPDisplay'; -import { getAuthCodes } from 'services/authenticator/authenticatorService'; +import { getAuthCodes } from 'services'; import { CustomError } from '@ente/shared/error'; import { PHOTOS_PAGES as PAGES } from '@ente/shared/constants/pages'; import { useRouter } from 'next/router'; diff --git a/apps/auth/src/services/authenticator/authenticatorService.ts b/apps/auth/src/services/index.ts similarity index 97% rename from apps/auth/src/services/authenticator/authenticatorService.ts rename to apps/auth/src/services/index.ts index ce3a145d0..e9160dd80 100644 --- a/apps/auth/src/services/authenticator/authenticatorService.ts +++ b/apps/auth/src/services/index.ts @@ -1,7 +1,7 @@ import { HttpStatusCode } from 'axios'; import HTTPService from '@ente/shared/network/HTTPService'; -import { AuthEntity, AuthKey } from 'types/authenticator/api'; -import { Code } from 'types/authenticator/code'; +import { AuthEntity, AuthKey } from 'types/api'; +import { Code } from 'types/code'; import ComlinkCryptoWorker from '@ente/shared/crypto'; import { getEndpoint } from '@ente/shared/network/api'; import { getActualKey } from '@ente/shared/user'; diff --git a/apps/auth/src/types/authenticator/api.ts b/apps/auth/src/types/api.ts similarity index 100% rename from apps/auth/src/types/authenticator/api.ts rename to apps/auth/src/types/api.ts diff --git a/apps/auth/src/types/authenticator/code.ts b/apps/auth/src/types/code.ts similarity index 100% rename from apps/auth/src/types/authenticator/code.ts rename to apps/auth/src/types/code.ts diff --git a/apps/photos/package.json b/apps/photos/package.json index dd50084de..3c5ecd652 100644 --- a/apps/photos/package.json +++ b/apps/photos/package.json @@ -21,7 +21,6 @@ "@tensorflow/tfjs-core": "^4.10.0", "@tensorflow/tfjs-tflite": "^0.0.1-alpha.7", "@zip.js/zip.js": "^2.4.2", - "axios": "^1.4.0", "bip39": "^3.0.4", "blazeface-back": "^0.0.9", "bootstrap": "^4.5.2", diff --git a/package.json b/package.json index 014fa45c8..deb30f199 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@ente/shared": "*", "@mui/icons-material": "5.14.1", "@mui/material": "5.11.16", + "axios": "^1.4.0", "is-electron": "^2.2.2", "next": "13.5.6", "react": "18.2.0", diff --git a/packages/accounts/api/srp.ts b/packages/accounts/api/srp.ts index e6247a5f9..496f8009c 100644 --- a/packages/accounts/api/srp.ts +++ b/packages/accounts/api/srp.ts @@ -13,7 +13,6 @@ import { UpdateSRPAndKeysRequest, UpdateSRPAndKeysResponse, } from '@ente/accounts/types/srp'; -import { getToken } from '@ente/shared/storage/localStorage/helpers'; import { ApiError, CustomError } from '@ente/shared/error'; import { HttpStatusCode } from 'axios'; import { logError } from '@ente/shared/sentry'; @@ -35,10 +34,10 @@ export const getSRPAttributes = async ( }; export const startSRPSetup = async ( + token: string, setupSRPRequest: SetupSRPRequest ): Promise => { try { - const token = getToken(); const resp = await HTTPService.post( `${ENDPOINT}/users/srp/setup`, setupSRPRequest, @@ -56,10 +55,10 @@ export const startSRPSetup = async ( }; export const completeSRPSetup = async ( + token: string, completeSRPSetupRequest: CompleteSRPSetupRequest ) => { try { - const token = getToken(); const resp = await HTTPService.post( `${ENDPOINT}/users/srp/complete`, completeSRPSetupRequest, diff --git a/packages/accounts/api/user.ts b/packages/accounts/api/user.ts index 61fc2ddb8..7bac4e05a 100644 --- a/packages/accounts/api/user.ts +++ b/packages/accounts/api/user.ts @@ -3,7 +3,7 @@ import { getEndpoint } from '@ente/shared/network/api'; import { getToken } from '@ente/shared/storage/localStorage/helpers'; import { KeyAttributes } from '@ente/shared/user/types'; -import { ApiError } from '@ente/shared/error'; +import { ApiError, CustomError } from '@ente/shared/error'; import { HttpStatusCode } from 'axios'; import { UserVerificationResponse, @@ -14,22 +14,22 @@ import { } from '@ente/accounts/types/user'; import { B64EncryptionResult } from '@ente/shared/crypto/types'; import { logError } from '@ente/shared/sentry'; +import { APPS, OTT_CLIENTS } from '@ente/shared/apps/constants'; const ENDPOINT = getEndpoint(); -export const sendOtt = (appName: string, email: string) => { +export const sendOtt = (appName: APPS, email: string) => { return HTTPService.post(`${ENDPOINT}/users/ott`, { email, - client: appName, + client: OTT_CLIENTS.get(appName), }); }; export const verifyOtt = (email: string, ott: string) => HTTPService.post(`${ENDPOINT}/users/verify-email`, { email, ott }); -export const putAttributes = async (keyAttributes: KeyAttributes) => { - const token = getToken(); - await HTTPService.put( +export const putAttributes = (token: string, keyAttributes: KeyAttributes) => + HTTPService.put( `${ENDPOINT}/users/attributes`, { keyAttributes }, undefined, @@ -37,23 +37,24 @@ export const putAttributes = async (keyAttributes: KeyAttributes) => { 'X-Auth-Token': token, } ); -}; export const _logout = async () => { - // ignore if token missing can be triggered during sign up. - if (!getToken()) return true; try { + const token = getToken(); await HTTPService.post(`${ENDPOINT}/users/logout`, null, undefined, { - 'X-Auth-Token': getToken(), + 'X-Auth-Token': token, }); - return true; } catch (e) { + // ignore if token missing can be triggered during sign up. + if (e instanceof Error && e.message === CustomError.TOKEN_MISSING) { + return; + } // ignore if unauthorized, can be triggered during on token expiry. - if ( + else if ( e instanceof ApiError && e.httpStatusCode === HttpStatusCode.Unauthorized ) { - return true; + return; } logError(e, '/users/logout failed'); throw e; @@ -88,9 +89,6 @@ export const removeTwoFactor = async (sessionID: string, secret: string) => { }; export const changeEmail = async (email: string, ott: string) => { - if (!getToken()) { - return null; - } await HTTPService.post( `${ENDPOINT}/users/change-email`, { @@ -105,9 +103,6 @@ export const changeEmail = async (email: string, ott: string) => { }; export const sendOTTForEmailChange = async (email: string) => { - if (!getToken()) { - return null; - } await HTTPService.post(`${ENDPOINT}/users/ott`, { email, client: 'web', diff --git a/packages/accounts/components/Login.tsx b/packages/accounts/components/Login.tsx index 6f11b7654..c6e095b76 100644 --- a/packages/accounts/components/Login.tsx +++ b/packages/accounts/components/Login.tsx @@ -12,10 +12,11 @@ import { Input } from '@mui/material'; import SingleInputForm, { SingleInputFormProps, } from '@ente/shared/components/SingleInputForm'; +import { APPS } from '@ente/shared/apps/constants'; interface LoginProps { signUp: () => void; - appName: string; + appName: APPS; } export default function Login(props: LoginProps) { diff --git a/packages/accounts/pages/change-password.tsx b/packages/accounts/pages/change-password.tsx index 775fbbe54..d717589a8 100644 --- a/packages/accounts/pages/change-password.tsx +++ b/packages/accounts/pages/change-password.tsx @@ -91,7 +91,7 @@ export default function ChangePassword({ appName, router }: PageProps) { const srpA = convertBufferToBase64(srpClient.computeA()); - const { setupID, srpB } = await startSRPSetup({ + const { setupID, srpB } = await startSRPSetup(token, { srpUserID, srpSalt, srpVerifier, diff --git a/packages/accounts/pages/credentials.tsx b/packages/accounts/pages/credentials.tsx index 355136f41..175df615d 100644 --- a/packages/accounts/pages/credentials.tsx +++ b/packages/accounts/pages/credentials.tsx @@ -137,9 +137,6 @@ export default function Credentials({ const getKeyAttributes: VerifyMasterPasswordFormProps['getKeyAttributes'] = async (kek: string) => { try { - if (!srpAttributes) { - throw Error('SRP attributes are missing'); - } const cryptoWorker = await ComlinkCryptoWorker.getInstance(); const response = await loginViaSRP(srpAttributes, kek); if (response.twoFactorSessionID) { @@ -176,10 +173,7 @@ export default function Credentials({ return keyAttributes; } } catch (e) { - if ( - e instanceof Error && - e.message !== CustomError.TWO_FACTOR_ENABLED - ) { + if (e.message !== CustomError.TWO_FACTOR_ENABLED) { logError(e, 'getKeyAttributes failed'); } throw e; diff --git a/packages/accounts/pages/generate.tsx b/packages/accounts/pages/generate.tsx index d293fe7ad..7641d4b69 100644 --- a/packages/accounts/pages/generate.tsx +++ b/packages/accounts/pages/generate.tsx @@ -31,6 +31,7 @@ import LinkButton from '@ente/shared/components/LinkButton'; import { PageProps } from '@ente/shared/apps/types'; export default function Generate({ router, appContext, appName }: PageProps) { + const [token, setToken] = useState(); const [user, setUser] = useState(); const [recoverModalView, setRecoveryModalView] = useState(false); const [loading, setLoading] = useState(true); @@ -54,6 +55,7 @@ export default function Generate({ router, appContext, appName }: PageProps) { } else if (keyAttributes?.encryptedKey) { router.push(PAGES.CREDENTIALS); } else { + setToken(user.token); setLoading(false); } }; @@ -66,7 +68,7 @@ export default function Generate({ router, appContext, appName }: PageProps) { const { keyAttributes, masterKey, srpSetupAttributes } = await generateKeyAndSRPAttributes(passphrase); - await putAttributes(keyAttributes); + await putAttributes(token, keyAttributes); await configureSRP(srpSetupAttributes); await generateAndSaveIntermediateKeyAttributes( passphrase, diff --git a/packages/accounts/pages/two-factor/recover.tsx b/packages/accounts/pages/two-factor/recover.tsx index 2682733b7..14e02ca1f 100644 --- a/packages/accounts/pages/two-factor/recover.tsx +++ b/packages/accounts/pages/two-factor/recover.tsx @@ -36,7 +36,6 @@ export default function Recover({ router, appContext }: PageProps) { useState(false); useEffect(() => { - router.prefetch(PAGES.GALLERY); const user = getData(LS_KEYS.USER); if (!user || !user.email || !user.twoFactorSessionID) { router.push(PAGES.ROOT); diff --git a/packages/accounts/pages/two-factor/setup.tsx b/packages/accounts/pages/two-factor/setup.tsx index 6ad35e317..bc12cc0f7 100644 --- a/packages/accounts/pages/two-factor/setup.tsx +++ b/packages/accounts/pages/two-factor/setup.tsx @@ -8,7 +8,6 @@ import VerifyTwoFactor, { } from '@ente/accounts/components/two-factor/VerifyForm'; import { encryptWithRecoveryKey } from '@ente/shared/crypto/helpers'; import { setData, LS_KEYS, getData } from '@ente/shared/storage/localStorage'; -import { PAGES } from '@ente/accounts/constants/pages'; import { TwoFactorSecret } from '@ente/accounts/types/user'; import Card from '@mui/material/Card'; import { Box, CardContent, Typography } from '@mui/material'; @@ -16,7 +15,7 @@ import { TwoFactorSetup } from '@ente/accounts/components/two-factor/setup'; import LinkButton from '@ente/shared/components/LinkButton'; import { PageProps } from '@ente/shared/apps/types'; import { logError } from '@ente/shared/sentry'; -import { APPS } from '@ente/shared/apps/constants'; +import { APP_HOMES } from '@ente/shared/apps/constants'; export enum SetupMode { QR_CODE, @@ -36,7 +35,6 @@ export default function SetupTwoFactor({ router, appName }: PageProps) { const twoFactorSecret = await setupTwoFactor(); setTwoFactorSecret(twoFactorSecret); } catch (e) { - console.log(e); logError(e, 'failed to get two factor setup code'); } }; @@ -56,11 +54,7 @@ export default function SetupTwoFactor({ router, appName }: PageProps) { ...getData(LS_KEYS.USER), isTwoFactorEnabled: true, }); - if (appName === APPS.AUTH) { - router.push(PAGES.AUTH); - } else { - router.push(PAGES.GALLERY); - } + router.push(APP_HOMES.get(appName)); }; return ( diff --git a/packages/accounts/pages/verify.tsx b/packages/accounts/pages/verify.tsx index 0383a68b8..fb8ddd2ce 100644 --- a/packages/accounts/pages/verify.tsx +++ b/packages/accounts/pages/verify.tsx @@ -99,6 +99,7 @@ export default function VerifyPage({ appContext, router, appName }: PageProps) { } else { if (getData(LS_KEYS.ORIGINAL_KEY_ATTRIBUTES)) { await putAttributes( + token, getData(LS_KEYS.ORIGINAL_KEY_ATTRIBUTES) ); } diff --git a/packages/accounts/services/srp.ts b/packages/accounts/services/srp.ts index 8da9072bb..7a4c65926 100644 --- a/packages/accounts/services/srp.ts +++ b/packages/accounts/services/srp.ts @@ -16,6 +16,7 @@ import { generateLoginSubKey } from '@ente/shared/crypto/helpers'; import { UserVerificationResponse } from '@ente/accounts/types/user'; import { logError } from '@ente/shared/sentry'; import { addLocalLog } from '@ente/shared/logging'; +import { getToken } from '@ente/shared/storage/localStorage/helpers'; const SRP_PARAMS = SRP.params['4096']; @@ -42,7 +43,8 @@ export const configureSRP = async ({ const srpA = convertBufferToBase64(srpClient.computeA()); addLocalLog(() => `srp a: ${srpA}`); - const { setupID, srpB } = await startSRPSetup({ + const token = getToken(); + const { setupID, srpB } = await startSRPSetup(token, { srpA, srpUserID, srpSalt, @@ -53,7 +55,7 @@ export const configureSRP = async ({ const srpM1 = convertBufferToBase64(srpClient.computeM1()); - const { srpM2 } = await completeSRPSetup({ + const { srpM2 } = await completeSRPSetup(token, { srpM1, setupID, }); diff --git a/packages/accounts/services/user.ts b/packages/accounts/services/user.ts index 32664b19f..2a6b4915e 100644 --- a/packages/accounts/services/user.ts +++ b/packages/accounts/services/user.ts @@ -3,11 +3,13 @@ import { _logout } from '../api/user'; import { PAGES } from '../constants/pages'; import { clearKeys } from '@ente/shared/storage/sessionStorage'; import { clearData } from '@ente/shared/storage/localStorage'; +import { deleteAllCache } from '@ente/shared/storage/cacheStorage/helpers'; import { logError } from '@ente/shared/sentry'; import { clearFiles } from '@ente/shared/storage/localForage/helpers'; import router from 'next/router'; -import ElectronAPIs from '@ente/shared/electron'; import isElectron from 'is-electron'; +import ElectronAPIs from '@ente/shared/electron'; +import { Events, eventBus } from '@ente/shared/events'; export const logoutUser = async () => { try { @@ -15,7 +17,6 @@ export const logoutUser = async () => { await _logout(); } catch (e) { // ignore - logError(e, 'clear InMemoryStore failed'); } try { InMemoryStore.clear(); @@ -33,11 +34,11 @@ export const logoutUser = async () => { } catch (e) { logError(e, 'clearData failed'); } - // try { - // await deleteAllCache(); - // } catch (e) { - // logError(e, 'deleteAllCache failed'); - // } + try { + await deleteAllCache(); + } catch (e) { + logError(e, 'deleteAllCache failed'); + } try { await clearFiles(); } catch (e) { @@ -50,11 +51,11 @@ export const logoutUser = async () => { logError(e, 'clearElectronStore failed'); } } - // try { - // eventBus.emit(Events.LOGOUT); - // } catch (e) { - // logError(e, 'Error in logout handlers'); - // } + try { + eventBus.emit(Events.LOGOUT); + } catch (e) { + logError(e, 'Error in logout handlers'); + } router.push(PAGES.ROOT); } catch (e) { logError(e, 'logoutUser failed'); diff --git a/packages/shared/apps/constants.ts b/packages/shared/apps/constants.ts index 4591e13a6..34016bf83 100644 --- a/packages/shared/apps/constants.ts +++ b/packages/shared/apps/constants.ts @@ -29,3 +29,8 @@ export const APP_HOMES = new Map([ [APPS.PHOTOS, PHOTOS_PAGES.GALLERY], [APPS.AUTH, AUTH_PAGES.AUTH], ]); + +export const OTT_CLIENTS = new Map([ + [APPS.PHOTOS, 'web'], + [APPS.AUTH, 'totp'], +]); diff --git a/packages/shared/electron/types.ts b/packages/shared/electron/types.ts index d1c6976cf..710db38e8 100644 --- a/packages/shared/electron/types.ts +++ b/packages/shared/electron/types.ts @@ -1,3 +1,7 @@ +import { LimitedCache } from '@ente/shared/storage/cacheStorage/types'; +import { ElectronFile } from '@ente/shared/upload/types'; +import { WatchMapping } from '@ente/shared/watchFolder/types'; + export interface AppUpdateInfo { autoUpdatable: boolean; version: string; @@ -14,18 +18,48 @@ export interface ElectronAPIsType { selectDirectory: () => Promise; sendNotification: (content: string) => void; readTextFile: (path: string) => Promise; + showUploadFilesDialog: () => Promise; + showUploadDirsDialog: () => Promise; + getPendingUploads: () => Promise<{ + files: ElectronFile[]; + collectionName: string; + type: string; + }>; setToUploadFiles: (type: string, filePaths: string[]) => void; + showUploadZipDialog: () => Promise<{ + zipPaths: string[]; + files: ElectronFile[]; + }>; + getElectronFilesFromGoogleZip: ( + filePath: string + ) => Promise; setToUploadCollection: (collectionName: string) => void; + getDirFiles: (dirPath: string) => Promise; + getWatchMappings: () => WatchMapping[]; + updateWatchMappingSyncedFiles: ( + folderPath: string, + files: WatchMapping['syncedFiles'] + ) => void; + updateWatchMappingIgnoredFiles: ( + folderPath: string, + files: WatchMapping['ignoredFiles'] + ) => void; addWatchMapping: ( collectionName: string, folderPath: string, uploadStrategy: number ) => Promise; removeWatchMapping: (folderPath: string) => Promise; + registerWatcherFunctions: ( + addFile: (file: ElectronFile) => Promise, + removeFile: (path: string) => Promise, + removeFolder: (folderPath: string) => Promise + ) => void; isFolder: (dirPath: string) => Promise; clearElectronStore: () => void; setEncryptionKey: (encryptionKey: string) => Promise; getEncryptionKey: () => Promise; + openDiskCache: (cacheName: string) => Promise; deleteDiskCache: (cacheName: string) => Promise; logToDisk: (msg: string) => void; convertToJPEG: ( @@ -40,7 +74,18 @@ export interface ElectronAPIsType { skipAppUpdate: (version: string) => void; getSentryUserID: () => Promise; getAppVersion: () => Promise; + runFFmpegCmd: ( + cmd: string[], + inputFile: File | ElectronFile, + outputFileName: string, + dontTimeout?: boolean + ) => Promise; muteUpdateNotification: (version: string) => void; + generateImageThumbnail: ( + inputFile: File | ElectronFile, + maxDimension: number, + maxSize: number + ) => Promise; logRendererProcessMemoryUsage: (message: string) => Promise; registerForegroundEventListener: (onForeground: () => void) => void; openDirectory: (dirPath: string) => Promise; @@ -49,4 +94,7 @@ export interface ElectronAPIsType { deleteFile: (path: string) => void; rename: (oldPath: string, newPath: string) => Promise; updateOptOutOfCrashReports: (optOut: boolean) => Promise; + computeImageEmbedding: (imageData: Uint8Array) => Promise; + computeTextEmbedding: (text: string) => Promise; + getPlatform: () => Promise<'mac' | 'windows' | 'linux'>; } diff --git a/packages/shared/events/index.ts b/packages/shared/events/index.ts new file mode 100644 index 000000000..670ecac47 --- /dev/null +++ b/packages/shared/events/index.ts @@ -0,0 +1,12 @@ +import { EventEmitter } from 'eventemitter3'; + +// When registering event handlers, +// handle errors to avoid unhandled rejection or propagation to emit call + +export enum Events { + LOGOUT = 'logout', + FILE_UPLOADED = 'fileUploaded', + LOCAL_FILES_UPDATED = 'localFilesUpdated', +} + +export const eventBus = new EventEmitter(); diff --git a/packages/shared/platform/index.ts b/packages/shared/platform/index.ts index ad34a00ff..60148c482 100644 --- a/packages/shared/platform/index.ts +++ b/packages/shared/platform/index.ts @@ -1,3 +1,13 @@ +import isElectron from 'is-electron'; + export function runningInBrowser() { return typeof window !== 'undefined'; } + +export function runningInWorker() { + return typeof importScripts === 'function'; +} + +export function runningInElectron() { + return isElectron(); +} diff --git a/packages/shared/sentry/utils.ts b/packages/shared/sentry/utils.ts index a8574f508..ec5308a9d 100644 --- a/packages/shared/sentry/utils.ts +++ b/packages/shared/sentry/utils.ts @@ -7,6 +7,8 @@ import isElectron from 'is-electron'; import { getAppEnv } from '@ente/shared/apps/env'; import { APP_ENV } from '@ente/shared/apps/constants'; import { isDisableSentryFlagSet } from '@ente/shared/apps/env'; +import { ApiError } from '../error'; +import { HttpStatusCode } from 'axios'; export async function getSentryUserID() { if (isElectron()) { @@ -37,7 +39,10 @@ function makeID(length) { export function isErrorUnnecessaryForSentry(error: any) { if (error?.message?.includes('Network Error')) { return true; - } else if (error?.status === 401) { + } else if ( + error instanceof ApiError && + error.httpStatusCode === HttpStatusCode.Unauthorized + ) { return true; } return false; diff --git a/packages/shared/storage/cacheStorage/constants.ts b/packages/shared/storage/cacheStorage/constants.ts new file mode 100644 index 000000000..80e0fecf1 --- /dev/null +++ b/packages/shared/storage/cacheStorage/constants.ts @@ -0,0 +1,5 @@ +export enum CACHES { + THUMBS = 'thumbs', + FACE_CROPS = 'face-crops', + FILES = 'files', +} diff --git a/packages/shared/storage/cacheStorage/factory.ts b/packages/shared/storage/cacheStorage/factory.ts new file mode 100644 index 000000000..7d4b5a969 --- /dev/null +++ b/packages/shared/storage/cacheStorage/factory.ts @@ -0,0 +1,48 @@ +import { LimitedCacheStorage } from './types'; +import { runningInElectron, runningInWorker } from '@ente/shared/platform'; +import { WorkerElectronCacheStorageService } from './workerElectron/service'; +import ElectronAPIs from '@ente/shared/electron'; + +class cacheStorageFactory { + workerElectronCacheStorageServiceInstance: WorkerElectronCacheStorageService; + getCacheStorage(): LimitedCacheStorage { + if (runningInElectron()) { + if (runningInWorker()) { + if (!this.workerElectronCacheStorageServiceInstance) { + this.workerElectronCacheStorageServiceInstance = + new WorkerElectronCacheStorageService(); + } + return this.workerElectronCacheStorageServiceInstance; + } else { + return { + open(cacheName) { + return ElectronAPIs.openDiskCache(cacheName); + }, + delete(cacheName) { + return ElectronAPIs.deleteDiskCache(cacheName); + }, + }; + } + } else { + return transformBrowserCacheStorageToLimitedCacheStorage(caches); + } + } +} + +export const CacheStorageFactory = new cacheStorageFactory(); + +function transformBrowserCacheStorageToLimitedCacheStorage( + caches: CacheStorage +): LimitedCacheStorage { + return { + async open(cacheName) { + const cache = await caches.open(cacheName); + return { + match: cache.match.bind(cache), + put: cache.put.bind(cache), + delete: cache.delete.bind(cache), + }; + }, + delete: caches.delete.bind(caches), + }; +} diff --git a/packages/shared/storage/cacheStorage/helpers.ts b/packages/shared/storage/cacheStorage/helpers.ts new file mode 100644 index 000000000..175627c7d --- /dev/null +++ b/packages/shared/storage/cacheStorage/helpers.ts @@ -0,0 +1,48 @@ +import { CACHES } from './constants'; +import { CacheStorageService } from '.'; +import { logError } from '@ente/shared/sentry'; + +export async function cached( + cacheName: string, + id: string, + get: () => Promise +): Promise { + const cache = await CacheStorageService.open(cacheName); + const cacheResponse = await cache.match(id); + + let result: Blob; + if (cacheResponse) { + result = await cacheResponse.blob(); + } else { + result = await get(); + + try { + await cache.put(id, new Response(result)); + } catch (e) { + // TODO: handle storage full exception. + console.error('Error while storing file to cache: ', id); + } + } + + return result; +} + +export async function getBlobFromCache( + cacheName: string, + url: string +): Promise { + const cache = await CacheStorageService.open(cacheName); + const response = await cache.match(url); + + return response.blob(); +} + +export async function deleteAllCache() { + try { + await CacheStorageService.delete(CACHES.THUMBS); + await CacheStorageService.delete(CACHES.FACE_CROPS); + await CacheStorageService.delete(CACHES.FILES); + } catch (e) { + logError(e, 'deleteAllCache failed'); // log and ignore + } +} diff --git a/packages/shared/storage/cacheStorage/index.ts b/packages/shared/storage/cacheStorage/index.ts new file mode 100644 index 000000000..9205c2aae --- /dev/null +++ b/packages/shared/storage/cacheStorage/index.ts @@ -0,0 +1,33 @@ +import { logError } from '@ente/shared/sentry'; +import { CacheStorageFactory } from './factory'; + +const SecurityError = 'SecurityError'; +const INSECURE_OPERATION = 'The operation is insecure.'; +async function openCache(cacheName: string) { + try { + return await CacheStorageFactory.getCacheStorage().open(cacheName); + } catch (e) { + // ignoring insecure operation error, as it is thrown in incognito mode in firefox + if (e.name === SecurityError && e.message === INSECURE_OPERATION) { + // no-op + } else { + // log and ignore, we don't want to break the caller flow, when cache is not available + logError(e, 'openCache failed'); + } + } +} +async function deleteCache(cacheName: string) { + try { + return await CacheStorageFactory.getCacheStorage().delete(cacheName); + } catch (e) { + // ignoring insecure operation error, as it is thrown in incognito mode in firefox + if (e.name === SecurityError && e.message === INSECURE_OPERATION) { + // no-op + } else { + // log and ignore, we don't want to break the caller flow, when cache is not available + logError(e, 'deleteCache failed'); + } + } +} + +export const CacheStorageService = { open: openCache, delete: deleteCache }; diff --git a/packages/shared/storage/cacheStorage/types.ts b/packages/shared/storage/cacheStorage/types.ts index e69de29bb..2920ece10 100644 --- a/packages/shared/storage/cacheStorage/types.ts +++ b/packages/shared/storage/cacheStorage/types.ts @@ -0,0 +1,20 @@ +export interface LimitedCacheStorage { + open: (cacheName: string) => Promise; + delete: (cacheName: string) => Promise; +} + +export interface LimitedCache { + match: (key: string) => Promise; + put: (key: string, data: Response) => Promise; + delete: (key: string) => Promise; +} + +export interface ProxiedLimitedCacheStorage { + open: (cacheName: string) => Promise; + delete: (cacheName: string) => Promise; +} +export interface ProxiedWorkerLimitedCache { + match: (key: string) => Promise; + put: (key: string, data: ArrayBuffer) => Promise; + delete: (key: string) => Promise; +} diff --git a/packages/shared/storage/cacheStorage/workerElectron/client.ts b/packages/shared/storage/cacheStorage/workerElectron/client.ts new file mode 100644 index 000000000..62414b15e --- /dev/null +++ b/packages/shared/storage/cacheStorage/workerElectron/client.ts @@ -0,0 +1,41 @@ +import * as Comlink from 'comlink'; +import { + LimitedCache, + ProxiedLimitedCacheStorage, + ProxiedWorkerLimitedCache, +} from '@ente/shared/storage/cacheStorage/types'; +import { serializeResponse, deserializeToResponse } from './utils/proxy'; +import ElectronAPIs from '@ente/shared/electron'; + +export class WorkerElectronCacheStorageClient + implements ProxiedLimitedCacheStorage +{ + async open(cacheName: string) { + const cache = await ElectronAPIs.openDiskCache(cacheName); + return Comlink.proxy({ + match: Comlink.proxy(transformMatch(cache.match.bind(cache))), + put: Comlink.proxy(transformPut(cache.put.bind(cache))), + delete: Comlink.proxy(cache.delete.bind(cache)), + }); + } + + async delete(cacheName: string) { + return await ElectronAPIs.deleteDiskCache(cacheName); + } +} + +function transformMatch( + fn: LimitedCache['match'] +): ProxiedWorkerLimitedCache['match'] { + return async (key: string) => { + return serializeResponse(await fn(key)); + }; +} + +function transformPut( + fn: LimitedCache['put'] +): ProxiedWorkerLimitedCache['put'] { + return async (key: string, data: ArrayBuffer) => { + fn(key, deserializeToResponse(data)); + }; +} diff --git a/packages/shared/storage/cacheStorage/workerElectron/service.ts b/packages/shared/storage/cacheStorage/workerElectron/service.ts new file mode 100644 index 000000000..c1fe50220 --- /dev/null +++ b/packages/shared/storage/cacheStorage/workerElectron/service.ts @@ -0,0 +1,55 @@ +import * as Comlink from 'comlink'; +import { + LimitedCache, + LimitedCacheStorage, + ProxiedWorkerLimitedCache, +} from '../types'; +import { WorkerElectronCacheStorageClient } from './client'; +import { wrap } from 'comlink'; +import { deserializeToResponse, serializeResponse } from './utils/proxy'; + +export class WorkerElectronCacheStorageService implements LimitedCacheStorage { + proxiedElectronCacheService: Comlink.Remote; + ready: Promise; + + constructor() { + this.ready = this.init(); + } + async init() { + const electronCacheStorageProxy = + wrap(self); + + this.proxiedElectronCacheService = + await new electronCacheStorageProxy(); + } + async open(cacheName: string) { + await this.ready; + const cache = await this.proxiedElectronCacheService.open(cacheName); + return { + match: transformMatch(cache.match.bind(cache)), + put: transformPut(cache.put.bind(cache)), + delete: cache.delete.bind(cache), + }; + } + + async delete(cacheName: string) { + await this.ready; + return await this.proxiedElectronCacheService.delete(cacheName); + } +} + +function transformMatch( + fn: ProxiedWorkerLimitedCache['match'] +): LimitedCache['match'] { + return async (key: string) => { + return deserializeToResponse(await fn(key)); + }; +} + +function transformPut( + fn: ProxiedWorkerLimitedCache['put'] +): LimitedCache['put'] { + return async (key: string, data: Response) => { + fn(key, await serializeResponse(data)); + }; +} diff --git a/packages/shared/storage/cacheStorage/workerElectron/utils/proxy.ts b/packages/shared/storage/cacheStorage/workerElectron/utils/proxy.ts new file mode 100644 index 000000000..d756cf1ec --- /dev/null +++ b/packages/shared/storage/cacheStorage/workerElectron/utils/proxy.ts @@ -0,0 +1,11 @@ +export function serializeResponse(response: Response) { + if (response) { + return response.arrayBuffer(); + } +} + +export function deserializeToResponse(arrayBuffer: ArrayBuffer) { + if (arrayBuffer) { + return new Response(arrayBuffer); + } +} diff --git a/packages/shared/upload/constants.ts b/packages/shared/upload/constants.ts new file mode 100644 index 000000000..aaa7f60d2 --- /dev/null +++ b/packages/shared/upload/constants.ts @@ -0,0 +1,4 @@ +export enum UPLOAD_STRATEGY { + SINGLE_COLLECTION, + COLLECTION_PER_FOLDER, +} diff --git a/packages/shared/watchFolder/types.ts b/packages/shared/watchFolder/types.ts new file mode 100644 index 000000000..8f16ffb39 --- /dev/null +++ b/packages/shared/watchFolder/types.ts @@ -0,0 +1,24 @@ +import { UPLOAD_STRATEGY } from '@ente/shared/upload/constants'; +import { ElectronFile } from '@ente/shared/upload/types'; + +export interface WatchMappingSyncedFile { + path: string; + uploadedFileID: number; + collectionID: number; +} + +export interface WatchMapping { + rootFolderName: string; + folderPath: string; + uploadStrategy: UPLOAD_STRATEGY; + syncedFiles: WatchMappingSyncedFile[]; + ignoredFiles: string[]; +} + +export interface EventQueueItem { + type: 'upload' | 'trash'; + folderPath: string; + collectionName?: string; + paths?: string[]; + files?: ElectronFile[]; +} diff --git a/yarn.lock b/yarn.lock index 95ba975bd..824d0d24b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2969,6 +2969,7 @@ __metadata: "@mui/icons-material": "npm:5.14.1" "@mui/material": "npm:5.11.16" "@typescript-eslint/parser": "npm:^5.59.2" + axios: "npm:^1.4.0" eslint: "npm:^8.28.0" husky: "npm:^7.0.1" is-electron: "npm:^2.2.2" @@ -6051,7 +6052,6 @@ __metadata: "@types/yup": "npm:^0.29.7" "@types/zxcvbn": "npm:^4.4.1" "@zip.js/zip.js": "npm:^2.4.2" - axios: "npm:^1.4.0" bip39: "npm:^3.0.4" blazeface-back: "npm:^0.0.9" bootstrap: "npm:^4.5.2"