Switch to castGateway API

apps/cast/src/constants/upload.ts

@@ -1,4 +1,4 @@
-import { ENCRYPTION_CHUNK_SIZE } from 'constants/crypto';
+import { ENCRYPTION_CHUNK_SIZE } from '@ente/shared/crypto/constants';
 import { FILE_TYPE } from 'constants/file';
 import {
     FileTypeInfo,

apps/cast/src/pages/index.tsx

@@ -150,7 +150,7 @@ export default function PairingMode() {
     const advertisePublicKey = async (publicKeyB64: string) => {
         // hey client, we exist!
         try {
-            await castGateway.advertisePublicKey(
+            await castGateway.registerDevice(
                 `${digits.join('')}`,
                 publicKeyB64
             );
@@ -171,7 +171,6 @@ export default function PairingMode() {
             const data = await pollForCastData();
 
             if (!data) return;
-
             storePayloadLocally(data);
             await router.push('/slideshow');
         }, 1000);

apps/cast/src/services/cast/castService.ts

@@ -0,0 +1,303 @@
+import { getEndpoint } from '@ente/shared/network/api';
+import localForage from '@ente/shared/storage/localForage';
+import { Collection, CollectionPublicMagicMetadata } from 'types/collection';
+import HTTPService from '@ente/shared/network/HTTPService';
+import { logError } from '@ente/shared/sentry';
+import { decryptFile, mergeMetadata, sortFiles } from 'utils/file';
+import { EncryptedEnteFile, EnteFile } from 'types/file';
+
+import { CustomError, parseSharingErrorCodes } from '@ente/shared/error';
+import ComlinkCryptoWorker from '@ente/shared/crypto';
+
+export interface SavedCollectionFiles {
+    collectionUID: string;
+    files: EnteFile[];
+}
+const ENDPOINT = getEndpoint();
+const COLLECTION_FILES_TABLE = 'collection-files';
+const COLLECTIONS_TABLE = 'collections';
+
+export const getPublicCollectionUID = (token: string) => `${token}`;
+
+const getLastSyncKey = (collectionUID: string) => `${collectionUID}-time`;
+
+export const getLocalFiles = async (
+    collectionUID: string
+): Promise<EnteFile[]> => {
+    const localSavedcollectionFiles =
+        (await localForage.getItem<SavedCollectionFiles[]>(
+            COLLECTION_FILES_TABLE
+        )) || [];
+    const matchedCollection = localSavedcollectionFiles.find(
+        (item) => item.collectionUID === collectionUID
+    );
+    return matchedCollection?.files || [];
+};
+export const savecollectionFiles = async (
+    collectionUID: string,
+    files: EnteFile[]
+) => {
+    const collectionFiles =
+        (await localForage.getItem<SavedCollectionFiles[]>(
+            COLLECTION_FILES_TABLE
+        )) || [];
+    await localForage.setItem(
+        COLLECTION_FILES_TABLE,
+        dedupeCollectionFiles([{ collectionUID, files }, ...collectionFiles])
+    );
+};
+
+export const getLocalCollections = async (collectionKey: string) => {
+    const localCollections =
+        (await localForage.getItem<Collection[]>(COLLECTIONS_TABLE)) || [];
+    const collection =
+        localCollections.find(
+            (localSavedPublicCollection) =>
+                localSavedPublicCollection.key === collectionKey
+        ) || null;
+    return collection;
+};
+
+export const saveCollection = async (collection: Collection) => {
+    const collections =
+        (await localForage.getItem<Collection[]>(COLLECTIONS_TABLE)) ?? [];
+    await localForage.setItem(
+        COLLECTIONS_TABLE,
+        dedupeCollections([collection, ...collections])
+    );
+};
+
+const dedupeCollections = (collections: Collection[]) => {
+    const keySet = new Set([]);
+    return collections.filter((collection) => {
+        if (!keySet.has(collection.key)) {
+            keySet.add(collection.key);
+            return true;
+        } else {
+            return false;
+        }
+    });
+};
+
+const dedupeCollectionFiles = (collectionFiles: SavedCollectionFiles[]) => {
+    const keySet = new Set([]);
+    return collectionFiles.filter(({ collectionUID }) => {
+        if (!keySet.has(collectionUID)) {
+            keySet.add(collectionUID);
+            return true;
+        } else {
+            return false;
+        }
+    });
+};
+
+async function getSyncTime(collectionUID: string): Promise<number> {
+    const lastSyncKey = getLastSyncKey(collectionUID);
+    const lastSyncTime = await localForage.getItem<number>(lastSyncKey);
+    return lastSyncTime ?? 0;
+}
+
+const updateSyncTime = async (collectionUID: string, time: number) =>
+    await localForage.setItem(getLastSyncKey(collectionUID), time);
+
+export const syncPublicFiles = async (
+    token: string,
+    collection: Collection,
+    setPublicFiles: (files: EnteFile[]) => void
+) => {
+    try {
+        let files: EnteFile[] = [];
+        const sortAsc = collection?.pubMagicMetadata?.data.asc ?? false;
+        const collectionUID = getPublicCollectionUID(token);
+        const localFiles = await getLocalFiles(collectionUID);
+
+        files = [...files, ...localFiles];
+        try {
+            if (!token) {
+                return sortFiles(files, sortAsc);
+            }
+            const lastSyncTime = await getSyncTime(collectionUID);
+            if (collection.updationTime === lastSyncTime) {
+                return sortFiles(files, sortAsc);
+            }
+            const fetchedFiles = await fetchFiles(
+                token,
+                collection,
+                lastSyncTime,
+                files,
+                setPublicFiles
+            );
+
+            files = [...files, ...fetchedFiles];
+            const latestVersionFiles = new Map<string, EnteFile>();
+            files.forEach((file) => {
+                const uid = `${file.collectionID}-${file.id}`;
+                if (
+                    !latestVersionFiles.has(uid) ||
+                    latestVersionFiles.get(uid).updationTime < file.updationTime
+                ) {
+                    latestVersionFiles.set(uid, file);
+                }
+            });
+            files = [];
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for (const [_, file] of latestVersionFiles) {
+                if (file.isDeleted) {
+                    continue;
+                }
+                files.push(file);
+            }
+            await savecollectionFiles(collectionUID, files);
+            await updateSyncTime(collectionUID, collection.updationTime);
+            setPublicFiles([...sortFiles(mergeMetadata(files), sortAsc)]);
+        } catch (e) {
+            const parsedError = parseSharingErrorCodes(e);
+            logError(e, 'failed to sync shared collection files');
+            if (parsedError.message === CustomError.TOKEN_EXPIRED) {
+                throw e;
+            }
+        }
+        return [...sortFiles(mergeMetadata(files), sortAsc)];
+    } catch (e) {
+        logError(e, 'failed to get local  or sync shared collection files');
+        throw e;
+    }
+};
+
+const fetchFiles = async (
+    token: string,
+    collection: Collection,
+    sinceTime: number,
+    files: EnteFile[],
+    setPublicFiles: (files: EnteFile[]) => void
+): Promise<EnteFile[]> => {
+    try {
+        let decryptedFiles: EnteFile[] = [];
+        let time = sinceTime;
+        let resp;
+        const sortAsc = collection?.pubMagicMetadata?.data.asc ?? false;
+        do {
+            if (!token) {
+                break;
+            }
+            resp = await HTTPService.get(
+                `${ENDPOINT}/public-collection/diff`,
+                {
+                    sinceTime: time,
+                },
+                {
+                    'Cache-Control': 'no-cache',
+                    'X-Auth-Access-Token': token,
+                }
+            );
+            decryptedFiles = [
+                ...decryptedFiles,
+                ...(await Promise.all(
+                    resp.data.diff.map(async (file: EncryptedEnteFile) => {
+                        if (!file.isDeleted) {
+                            return await decryptFile(file, collection.key);
+                        } else {
+                            return file;
+                        }
+                    }) as Promise<EnteFile>[]
+                )),
+            ];
+
+            if (resp.data.diff.length) {
+                time = resp.data.diff.slice(-1)[0].updationTime;
+            }
+            setPublicFiles(
+                sortFiles(
+                    mergeMetadata(
+                        [...(files || []), ...decryptedFiles].filter(
+                            (item) => !item.isDeleted
+                        )
+                    ),
+                    sortAsc
+                )
+            );
+        } while (resp.data.hasMore);
+        return decryptedFiles;
+    } catch (e) {
+        logError(e, 'Get public  files failed');
+        throw e;
+    }
+};
+
+export const getCastCollection = async (
+    token: string,
+    collectionKey: string
+): Promise<[Collection]> => {
+    try {
+        if (!token) {
+            return;
+        }
+        const resp = await HTTPService.get(
+            `${ENDPOINT}/public-collection/info`,
+            null,
+            { 'Cache-Control': 'no-cache', 'X-Auth-Access-Token': token }
+        );
+        const fetchedCollection = resp.data.collection;
+
+        const cryptoWorker = await ComlinkCryptoWorker.getInstance();
+
+        const collectionName = (fetchedCollection.name =
+            fetchedCollection.name ||
+            (await cryptoWorker.decryptToUTF8(
+                fetchedCollection.encryptedName,
+                fetchedCollection.nameDecryptionNonce,
+                collectionKey
+            )));
+
+        let collectionPublicMagicMetadata: CollectionPublicMagicMetadata;
+        if (fetchedCollection.pubMagicMetadata?.data) {
+            collectionPublicMagicMetadata = {
+                ...fetchedCollection.pubMagicMetadata,
+                data: await cryptoWorker.decryptMetadata(
+                    fetchedCollection.pubMagicMetadata.data,
+                    fetchedCollection.pubMagicMetadata.header,
+                    collectionKey
+                ),
+            };
+        }
+
+        const collection = {
+            ...fetchedCollection,
+            name: collectionName,
+            key: collectionKey,
+            pubMagicMetadata: collectionPublicMagicMetadata,
+        };
+        await saveCollection(collection);
+        return [collection];
+    } catch (e) {
+        logError(e, 'failed to get public collection');
+        throw e;
+    }
+};
+
+export const removeCollection = async (
+    collectionUID: string,
+    collectionKey: string
+) => {
+    const collections =
+        (await localForage.getItem<Collection[]>(COLLECTIONS_TABLE)) || [];
+    await localForage.setItem(
+        COLLECTIONS_TABLE,
+        collections.filter((collection) => collection.key !== collectionKey)
+    );
+    await removeCollectionFiles(collectionUID);
+};
+
+export const removeCollectionFiles = async (collectionUID: string) => {
+    await localForage.removeItem(getLastSyncKey(collectionUID));
+    const collectionFiles =
+        (await localForage.getItem<SavedCollectionFiles[]>(
+            COLLECTION_FILES_TABLE
+        )) ?? [];
+    await localForage.setItem(
+        COLLECTION_FILES_TABLE,
+        collectionFiles.filter(
+            (collectionFiles) => collectionFiles.collectionUID !== collectionUID
+        )
+    );
+};

apps/cast/src/services/castDownloadManager.ts

@@ -18,7 +18,7 @@ import {
 import HTTPService from '@ente/shared/network/HTTPService';
 import { logError } from '@ente/shared/sentry';
 
-class PublicCollectionDownloadManager {
+class CastDownloadManager {
     private fileObjectURLPromise = new Map<
         string,
         Promise<{ original: string[]; converted: string[] }>
@@ -311,4 +311,4 @@ class PublicCollectionDownloadManager {
     };
 }
 
-export default new PublicCollectionDownloadManager();
+export default new CastDownloadManager();

apps/cast/src/services/collectionService.ts

@@ -1,7 +1,6 @@
 import { getActualKey } from '@ente/shared/user';
 import { batch } from '@ente/shared/batch';
 import { EnteFile } from 'types/file';
-import { CustomError } from '@ente/shared/error';
 import {
     sortFiles,
     groupFilesBasedOnCollectionID,
@@ -44,7 +43,6 @@ import {
     isPinnedCollection,
     updateMagicMetadata,
 } from 'utils/magicMetadata';
-import { FamilyData, User } from 'types/user';
 import {
     isQuickLinkCollection,
     isOutgoingShare,
@@ -70,6 +68,7 @@ import HTTPService from '@ente/shared/network/HTTPService';
 import { logError } from '@ente/shared/sentry';
 import { LS_KEYS, getData } from '@ente/shared/storage/localStorage';
 import localForage from '@ente/shared/storage/localForage';
+import { User } from '@ente/shared/user/types';
 
 const ENDPOINT = getEndpoint();
 const COLLECTION_TABLE = 'collections';
@@ -78,7 +77,6 @@ const HIDDEN_COLLECTION_IDS = 'hidden-collection-ids';
 
 const UNCATEGORIZED_COLLECTION_NAME = 'Uncategorized';
 export const HIDDEN_COLLECTION_NAME = '.hidden';
-const FAVORITE_COLLECTION_NAME = 'Favorites';
 
 export const getCollectionLastSyncTime = async (collection: Collection) =>
     (await localForage.getItem<number>(`${collection.id}-time`)) ?? 0;
@@ -516,34 +514,6 @@ const postCollection = async (
     }
 };
 
-export const createFavoritesCollection = () => {
-    return createCollection(FAVORITE_COLLECTION_NAME, CollectionType.favorites);
-};
-
-export const addToFavorites = async (file: EnteFile) => {
-    try {
-        let favCollection = await getFavCollection();
-        if (!favCollection) {
-            favCollection = await createFavoritesCollection();
-        }
-        await addToCollection(favCollection, [file]);
-    } catch (e) {
-        logError(e, 'failed to add to favorite');
-    }
-};
-
-export const removeFromFavorites = async (file: EnteFile) => {
-    try {
-        const favCollection = await getFavCollection();
-        if (!favCollection) {
-            throw Error(CustomError.FAV_COLLECTION_MISSING);
-        }
-        await removeFromCollection(favCollection.id, [file]);
-    } catch (e) {
-        logError(e, 'remove from favorite failed');
-    }
-};
-
 export const addToCollection = async (
     collection: Collection,
     files: EnteFile[]
@@ -1395,77 +1365,3 @@ export async function moveToHiddenCollection(files: EnteFile[]) {
         throw e;
     }
 }
-
-export async function unhideToCollection(
-    collection: Collection,
-    files: EnteFile[]
-) {
-    try {
-        const groupiedFiles = groupFilesBasedOnCollectionID(files);
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        for (const [collectionID, files] of groupiedFiles.entries()) {
-            if (collectionID === collection.id) {
-                continue;
-            }
-            await moveToCollection(collectionID, collection, files);
-        }
-    } catch (e) {
-        logError(e, 'unhide to collection failed ');
-        throw e;
-    }
-}
-
-export const constructUserIDToEmailMap = (
-    user: User,
-    collections: Collection[]
-): Map<number, string> => {
-    try {
-        const userIDToEmailMap = new Map<number, string>();
-        collections.forEach((item) => {
-            const { owner, sharees } = item;
-            if (user.id !== owner.id && owner.email) {
-                userIDToEmailMap.set(owner.id, owner.email);
-            }
-            if (sharees) {
-                sharees.forEach((item) => {
-                    if (item.id !== user.id)
-                        userIDToEmailMap.set(item.id, item.email);
-                });
-            }
-        });
-        return userIDToEmailMap;
-    } catch (e) {
-        logError('Error Mapping UserId to email:', e);
-        return new Map<number, string>();
-    }
-};
-
-export const constructEmailList = (
-    user: User,
-    collections: Collection[],
-    familyData: FamilyData
-): string[] => {
-    const emails = collections
-        .map((item) => {
-            const { owner, sharees } = item;
-            if (owner.email && item.owner.id !== user.id) {
-                return [item.owner.email];
-            } else {
-                if (!sharees?.length) {
-                    return [];
-                }
-                const shareeEmails = item.sharees
-                    .filter((sharee) => sharee.email !== user.email)
-                    .map((sharee) => sharee.email);
-                return shareeEmails;
-            }
-        })
-        .flat();
-
-    // adding family members
-    if (familyData) {
-        const family = familyData.members.map((member) => member.email);
-        emails.push(...family);
-    }
-    return Array.from(new Set(emails));
-};

apps/cast/src/types/collection/index.ts

@@ -58,6 +58,9 @@ export interface Collection
     sharedMagicMetadata: CollectionShareeMagicMetadata;
 }
 
+// define a method on Collection interface to return the sync key as collection.id-time
+// this is used to store the last sync time of a collection in local storage
+
 export interface PublicURL {
     url: string;
     deviceLimit: number;

apps/cast/src/utils/collection/index.ts

@@ -2,7 +2,6 @@ import {
     getNonEmptyCollections,
     updateCollectionMagicMetadata,
     updatePublicCollectionMagicMetadata,
-    updateSharedCollectionMagicMetadata,
 } from 'services/collectionService';
 import { EnteFile } from 'types/file';
 import { logError } from '@ente/shared/sentry';
@@ -99,44 +98,6 @@ export const shareExpiryOptions = () => [
     },
 ];
 
-export const changeCollectionVisibility = async (
-    collection: Collection,
-    visibility: VISIBILITY_STATE
-) => {
-    try {
-        const updatedMagicMetadataProps: CollectionMagicMetadataProps = {
-            visibility,
-        };
-
-        const user: User = getData(LS_KEYS.USER);
-        if (collection.owner.id === user.id) {
-            const updatedMagicMetadata = await updateMagicMetadata(
-                updatedMagicMetadataProps,
-                collection.magicMetadata,
-                collection.key
-            );
-
-            await updateCollectionMagicMetadata(
-                collection,
-                updatedMagicMetadata
-            );
-        } else {
-            const updatedMagicMetadata = await updateMagicMetadata(
-                updatedMagicMetadataProps,
-                collection.sharedMagicMetadata,
-                collection.key
-            );
-            await updateSharedCollectionMagicMetadata(
-                collection,
-                updatedMagicMetadata
-            );
-        }
-    } catch (e) {
-        logError(e, 'change collection visibility failed');
-        throw e;
-    }
-};
-
 export const changeCollectionSortOrder = async (
     collection: Collection,
     asc: boolean

apps/cast/src/utils/file/index.ts

@@ -21,7 +21,7 @@ import {
     SUPPORTED_RAW_FORMATS,
     RAW_FORMATS,
 } from 'constants/file';
-import PublicCollectionDownloadManager from 'services/publicCollectionDownloadManager';
+import CastDownloadManager from 'services/castDownloadManager';
 import heicConversionService from 'services/heicConversionService';
 import * as ffmpegService from 'services/ffmpeg/ffmpegService';
 import { VISIBILITY_STATE } from 'types/magicMetadata';
@@ -96,13 +96,12 @@ export async function downloadFile(
         let fileBlob: Blob;
         const fileReader = new FileReader();
         if (accessedThroughSharedURL) {
-            const fileURL =
-                await PublicCollectionDownloadManager.getCachedOriginalFile(
-                    file
-                )[0];
+            const fileURL = await CastDownloadManager.getCachedOriginalFile(
+                file
+            )[0];
             if (!fileURL) {
                 fileBlob = await new Response(
-                    await PublicCollectionDownloadManager.downloadFile(
+                    await CastDownloadManager.downloadFile(
                         token,
                         passwordToken,
                         file

apps/photos/src/components/Collections/CollectionOptions/AlbumCastDialog.tsx

@@ -4,7 +4,8 @@ import SingleInputForm, {
     SingleInputFormProps,
 } from '@ente/shared/components/SingleInputForm';
 import { t } from 'i18next';
-import { getKexValue, setKexValue } from '@ente/shared/network/kexService';
+import { v4 as uuidv4 } from 'uuid';
+import castGateway from '@ente/shared/network/cast';
 import { boxSeal, toB64 } from '@ente/shared/crypto/internal/libsodium';
 import { loadSender } from '@ente/shared/hooks/useCastSender';
 import { useEffect, useState } from 'react';
@@ -13,7 +14,6 @@ import EnteSpinner from '@ente/shared/components/EnteSpinner';
 import { VerticallyCentered } from '@ente/shared/components/Container';
 import { logError } from '@ente/shared/sentry';
 import { Collection } from 'types/collection';
-import { getToken } from '@ente/shared/storage/localStorage/helpers';
 
 interface Props {
     show: boolean;
@@ -63,16 +63,16 @@ export default function AlbumCastDialog(props: Props) {
 
     const doCast = async (pin: string) => {
         // does the TV exist? have they advertised their existence?
-        const tvPublicKeyKexKey = `${pin}_pubkey`;
-
-        const tvPublicKeyB64 = await getKexValue(tvPublicKeyKexKey);
+        const tvPublicKeyB64 = await castGateway.getPublicKey(pin);
         if (!tvPublicKeyB64) {
             throw new Error(AlbumCastError.TV_NOT_FOUND);
         }
+        // generate random uuid string
+        const castToken = uuidv4();
 
         // ok, they exist. let's give them the good stuff.
         const payload = JSON.stringify({
-            castToken: getToken(),
+            castToken: castToken,
             collectionID: props.currentCollection.id,
             collectionKey: props.currentCollection.key,
         });
@@ -82,10 +82,13 @@ export default function AlbumCastDialog(props: Props) {
             tvPublicKeyB64
         );
 
-        const encryptedPayloadForTvKexKey = `${pin}_payload`;
-
         // hey TV, we acknowlege you!
-        await setKexValue(encryptedPayloadForTvKexKey, encryptedPayload);
+        await castGateway.publishCastPayload(
+            pin,
+            encryptedPayload,
+            props.currentCollection.id,
+            castToken
+        );
     };
 
     useEffect(() => {

packages/shared/network/cast.ts

@@ -1,4 +1,5 @@
 import { logError } from '../sentry';
+import { getToken } from '../storage/localStorage/helpers';
 import HTTPService from './HTTPService';
 import { getEndpoint } from './api';
 
@@ -8,41 +9,59 @@ class CastGateway {
     public async getCastData(code: string): Promise<string> {
         let resp;
         try {
-            resp = await HTTPService.get(`${getEndpoint()}/kex/get`, {
-                identifier: `${code}_payload`,
-            });
+            resp = await HTTPService.get(
+                `${getEndpoint()}/cast/cast-data/${code}`
+            );
         } catch (e) {
             logError(e, 'failed to getCastData');
             throw e;
         }
-        return resp.data.wrappedKey;
+        return resp.data.encPayload;
     }
 
     public async getPublicKey(code: string): Promise<string> {
         let resp;
         try {
-            resp = await HTTPService.get(`${getEndpoint()}/kex/get`, {
-                identifier: `${code}_pubkey`,
-            });
+            const token = getToken();
+            resp = await HTTPService.get(
+                `${getEndpoint()}/cast/device-info/${code}`,
+                undefined,
+                {
+                    'X-Auth-Token': token,
+                }
+            );
         } catch (e) {
             logError(e, 'failed to getPublicKey');
             throw e;
         }
-        return resp.data.wrappedKey;
+        return resp.data.publicKey;
     }
 
-    public async advertisePublicKey(code: string, publicKey: string) {
-        await HTTPService.put(getEndpoint() + '/kex/add', {
-            customIdentifier: `${code}_pubkey`,
-            wrappedKey: publicKey,
+    public async registerDevice(code: string, publicKey: string) {
+        await HTTPService.put(getEndpoint() + '/cast/device-info/', {
+            deviceCode: `${code}`,
+            publicKey: publicKey,
         });
     }
 
-    public async publishCastPayload(code: string, castPayload: string) {
-        await HTTPService.put(getEndpoint() + '/kex/add', {
-            customIdentifier: `${code}_payload`,
-            wrappedKey: castPayload,
-        });
+    public async publishCastPayload(
+        code: string,
+        castPayload: string,
+        collectionID: number,
+        castToken: string
+    ) {
+        const token = getToken();
+        await HTTPService.post(
+            getEndpoint() + '/cast/cast-data/',
+            {
+                deviceCode: `${code}`,
+                encPayload: castPayload,
+                collectionID: collectionID,
+                castToken: castToken,
+            },
+            undefined,
+            { 'X-Auth-Token': token }
+        );
     }
 }