[web] Cast ready to roll (#1671)

desktop/src/main/log.ts

@@ -62,7 +62,7 @@ const logError = (message: string, e?: unknown) => {
 
 const logError_ = (message: string) => {
     log.error(`[main] [error] ${message}`);
-    if (isDev) console.error(`[error] ${message}`);
+    console.error(`[error] ${message}`);
 };
 
 const logInfo = (...params: unknown[]) => {
@@ -96,8 +96,8 @@ export default {
      * any arbitrary object that we obtain, say, when in a try-catch handler (in
      * JavaScript any arbitrary value can be thrown).
      *
-     * The log is written to disk. In development builds, the log is also
-     * printed to the main (Node.js) process console.
+     * The log is written to disk and printed to the main (Node.js) process's
+     * console.
      */
     error: logError,
     /**
@@ -120,7 +120,7 @@ export default {
      * The function can return an arbitrary value which is serialized before
      * being logged.
      *
-     * This log is NOT written to disk. And it is printed to the main (Node.js)
+     * This log is NOT written to disk. It is printed to the main (Node.js)
      * process console, but only on development builds.
      */
     debug: logDebug,

web/apps/cast/src/components/Slide.tsx

@@ -1,56 +0,0 @@
-interface SlideViewProps {
-    /** The URL of the image to show. */
-    url: string;
-    /** The URL of the next image that we will transition to. */
-    nextURL: string;
-}
-
-/**
- * Show the image at {@link url} in a full screen view.
- *
- * Also show {@link nextURL} in an hidden image view to prepare the browser for
- * an imminent transition to it.
- */
-export const SlideView: React.FC<SlideViewProps> = ({ url, nextURL }) => {
-    return (
-        <div
-            style={{
-                width: "100vw",
-                height: "100vh",
-                backgroundImage: `url(${url})`,
-                backgroundSize: "cover",
-                backgroundPosition: "center",
-                backgroundRepeat: "no-repeat",
-                backgroundBlendMode: "multiply",
-                backgroundColor: "rgba(0, 0, 0, 0.5)",
-            }}
-        >
-            <div
-                style={{
-                    height: "100%",
-                    width: "100%",
-                    display: "flex",
-                    justifyContent: "center",
-                    alignItems: "center",
-                    backdropFilter: "blur(10px)",
-                }}
-            >
-                <img
-                    src={nextURL}
-                    style={{
-                        maxWidth: "100%",
-                        maxHeight: "100%",
-                        display: "none",
-                    }}
-                />
-                <img
-                    src={url}
-                    style={{
-                        maxWidth: "100%",
-                        maxHeight: "100%",
-                    }}
-                />
-            </div>
-        </div>
-    );
-};

web/apps/cast/src/pages/_app.tsx

@@ -1,4 +1,5 @@
 import { CustomHead } from "@/next/components/Head";
+import { disableDiskLogs } from "@/next/log";
 import { logUnhandledErrorsAndRejections } from "@/next/log-web";
 import { APPS, APP_TITLES } from "@ente/shared/apps/constants";
 import { getTheme } from "@ente/shared/themes";
@@ -11,6 +12,7 @@ import "styles/global.css";
 
 export default function App({ Component, pageProps }: AppProps) {
     useEffect(() => {
+        disableDiskLogs();
         logUnhandledErrorsAndRejections(true);
         return () => logUnhandledErrorsAndRejections(false);
     }, []);

web/apps/cast/src/pages/index.tsx

@@ -4,19 +4,15 @@ import { styled } from "@mui/material";
 import { PairingCode } from "components/PairingCode";
 import { useRouter } from "next/router";
 import { useEffect, useState } from "react";
-import { storeCastData } from "services/cast";
-import { advertiseCode, getCastData, register } from "services/pair";
-import { castReceiverLoadingIfNeeded } from "../utils/cast-receiver";
+import { readCastData, storeCastData } from "services/cast-data";
+import { getCastData, register } from "services/pair";
+import { advertiseOnChromecast } from "../services/chromecast";
 
 export default function Index() {
     const [publicKeyB64, setPublicKeyB64] = useState<string | undefined>();
     const [privateKeyB64, setPrivateKeyB64] = useState<string | undefined>();
     const [pairingCode, setPairingCode] = useState<string | undefined>();
 
-    // Keep a boolean flag to ensure that Cast Receiver starts only once even if
-    // pairing codes change.
-    const [haveInitializedCast, setHaveInitializedCast] = useState(false);
-
     const router = useRouter();
 
     useEffect(() => {
@@ -27,12 +23,10 @@ export default function Index() {
                 setPairingCode(r.pairingCode);
             });
         } else {
-            if (!haveInitializedCast) {
-                castReceiverLoadingIfNeeded().then((cast) => {
-                    setHaveInitializedCast(true);
-                    advertiseCode(cast, () => pairingCode);
-                });
-            }
+            advertiseOnChromecast(
+                () => pairingCode,
+                () => readCastData()?.collectionID,
+            );
         }
     }, [pairingCode]);
 
@@ -52,7 +46,6 @@ export default function Index() {
                 return;
             }
 
-            log.info("Pairing complete");
             storeCastData(data);
             await router.push("/slideshow");
         } catch (e) {

web/apps/cast/src/pages/slideshow.tsx

@@ -1,15 +1,16 @@
 import log from "@/next/log";
+import { ensure } from "@/utils/ensure";
 import { styled } from "@mui/material";
 import { FilledCircleCheck } from "components/FilledCircleCheck";
-import { SlideView } from "components/Slide";
 import { useRouter } from "next/router";
 import { useEffect, useState } from "react";
-import { readCastData, renderableImageURLs } from "services/cast";
+import { readCastData } from "services/cast-data";
+import { isChromecast } from "services/chromecast";
+import { imageURLGenerator } from "services/render";
 
 export default function Slideshow() {
     const [loading, setLoading] = useState(true);
     const [imageURL, setImageURL] = useState<string | undefined>();
-    const [nextImageURL, setNextImageURL] = useState<string | undefined>();
     const [isEmpty, setIsEmpty] = useState(false);
 
     const router = useRouter();
@@ -22,19 +23,18 @@ export default function Slideshow() {
 
         const loop = async () => {
             try {
-                const urlGenerator = renderableImageURLs(readCastData());
+                const urlGenerator = imageURLGenerator(ensure(readCastData()));
                 while (!stop) {
-                    const { value: urls, done } = await urlGenerator.next();
-                    if (done) {
+                    const { value: url, done } = await urlGenerator.next();
+                    if (done || !url) {
                         // No items in this callection can be shown.
                         setIsEmpty(true);
-                        // Go back to pairing screen after 3 seconds.
+                        // Go back to pairing screen after 5 seconds.
                         setTimeout(pair, 5000);
                         return;
                     }
 
-                    setImageURL(urls[0]);
-                    setNextImageURL(urls[1]);
+                    setImageURL(url);
                     setLoading(false);
                 }
             } catch (e) {
@@ -50,12 +50,14 @@ export default function Slideshow() {
         };
     }, []);
 
-    console.log("Rendering slideshow", { loading, imageURL, nextImageURL });
-
     if (loading) return <PairingComplete />;
     if (isEmpty) return <NoItems />;
 
-    return <SlideView url={imageURL} nextURL={nextImageURL} />;
+    return isChromecast() ? (
+        <SlideViewChromecast url={imageURL} />
+    ) : (
+        <SlideView url={imageURL} />
+    );
 }
 
 const PairingComplete: React.FC = () => {
@@ -71,19 +73,13 @@ const PairingComplete: React.FC = () => {
     );
 };
 
-const Message: React.FC<React.PropsWithChildren> = ({ children }) => {
-    return (
-        <Message_>
-            <MessageItems>{children}</MessageItems>
-        </Message_>
-    );
-};
-
-const Message_ = styled("div")`
+const Message = styled("div")`
     display: flex;
-    min-height: 100svh;
+    flex-direction: column;
+    height: 100%;
     justify-content: center;
     align-items: center;
+    text-align: center;
 
     line-height: 1.5rem;
 
@@ -92,13 +88,6 @@ const Message_ = styled("div")`
     }
 `;
 
-const MessageItems = styled("div")`
-    display: flex;
-    flex-direction: column;
-    align-items: center;
-    text-align: center;
-`;
-
 const NoItems: React.FC = () => {
     return (
         <Message>
@@ -110,3 +99,94 @@ const NoItems: React.FC = () => {
         </Message>
     );
 };
+
+interface SlideViewProps {
+    /** The URL of the image to show. */
+    url: string;
+}
+
+const SlideView: React.FC<SlideViewProps> = ({ url }) => {
+    return (
+        <SlideView_ style={{ backgroundImage: `url(${url})` }}>
+            <img src={url} decoding="sync" alt="" />
+        </SlideView_>
+    );
+};
+
+const SlideView_ = styled("div")`
+    width: 100%;
+    height: 100%;
+
+    background-size: cover;
+    background-position: center;
+    background-repeat: no-repeat;
+    background-blend-mode: multiply;
+    background-color: rgba(0, 0, 0, 0.5);
+
+    /* Smooth out the transition a bit.
+     *
+     * For the img itself, we set decoding="sync" to have it switch seamlessly.
+     * But there does not seem to be a way of setting decoding sync for the
+     * background image, and for large (multi-MB) images the background image
+     * switch is still visually non-atomic.
+     *
+     * As a workaround, add a long transition so that the background image
+     * transitions in a more "fade-to" manner. This effect might or might not be
+     * visually the best though.
+     *
+     * Does not work in Firefox, but that's fine, this is only a slight tweak,
+     * not a functional requirement.
+     */
+    transition: all 2s;
+
+    img {
+        width: 100%;
+        height: 100%;
+        backdrop-filter: blur(10px);
+        object-fit: contain;
+    }
+`;
+
+/**
+ * Variant of {@link SlideView} for use when we're running on Chromecast.
+ *
+ * Chromecast devices have trouble with
+ *
+ *     backdrop-filter: blur(10px);
+ *
+ * So emulate a cheaper approximation for use on Chromecast.
+ */
+const SlideViewChromecast: React.FC<SlideViewProps> = ({ url }) => {
+    return (
+        <SlideViewChromecast_>
+            <img className="svc-bg" src={url} alt="" />
+            <img className="svc-content" src={url} decoding="sync" alt="" />
+        </SlideViewChromecast_>
+    );
+};
+
+const SlideViewChromecast_ = styled("div")`
+    width: 100%;
+    height: 100%;
+
+    /* We can't set opacity of background-image, so use a wrapper */
+    position: relative;
+    overflow: hidden;
+
+    img.svc-bg {
+        position: absolute;
+        left: 0;
+        top: 0;
+        width: 100%;
+        height: 100%;
+        object-fit: cover;
+        opacity: 0.1;
+    }
+
+    img.svc-content {
+        position: relative;
+        width: 100%;
+        height: 100%;
+        object-fit: contain;
+    }
+`;

web/apps/cast/src/services/cast-data.ts

@@ -0,0 +1,41 @@
+export interface CastData {
+    /** The ID of the callection we are casting. */
+    collectionID: string;
+    /** A key to decrypt the collection we are casting. */
+    collectionKey: string;
+    /** A credential to use for fetching media files for this cast session. */
+    castToken: string;
+}
+
+/**
+ * Save the data received after pairing with a sender into local storage.
+ *
+ * We will read in back when we start the slideshow.
+ */
+export const storeCastData = (payload: unknown) => {
+    if (!payload || typeof payload != "object")
+        throw new Error("Unexpected cast data");
+
+    // Iterate through all the keys of the payload object and save them to
+    // localStorage. We don't validate here, we'll validate when we read these
+    // values back in `readCastData`.
+    for (const key in payload) {
+        window.localStorage.setItem(key, payload[key]);
+    }
+};
+
+/**
+ * Read back the cast data we got after pairing.
+ *
+ * Sibling of {@link storeCastData}. It returns undefined if the expected data
+ * is not present in localStorage.
+ */
+export const readCastData = (): CastData | undefined => {
+    const collectionID = localStorage.getItem("collectionID");
+    const collectionKey = localStorage.getItem("collectionKey");
+    const castToken = localStorage.getItem("castToken");
+
+    return collectionID && collectionKey && castToken
+        ? { collectionID, collectionKey, castToken }
+        : undefined;
+};

web/apps/cast/src/services/chromecast.ts

@@ -0,0 +1,227 @@
+/// <reference types="chromecast-caf-receiver" />
+
+import log from "@/next/log";
+
+export type Cast = typeof cast;
+
+/**
+ * A holder for the "cast" global object exposed by the Chromecast SDK,
+ * alongwith auxiliary state we need around it.
+ */
+class CastReceiver {
+    /**
+     * A reference to the `cast` global object that the Chromecast Web Receiver
+     * SDK attaches to the window.
+     *
+     * https://developers.google.com/cast/docs/web_receiver/basic
+     */
+    cast: Cast | undefined;
+    /**
+     * A promise that allows us to ensure multiple requests to load are funneled
+     * through the same reified load.
+     */
+    loader: Promise<Cast> | undefined;
+    /**
+     * True if we have already attached listeners (i.e. if we have "started" the
+     * Chromecast SDK).
+     *
+     * Note that "stopping" the Chromecast SDK causes the Chromecast device to
+     * reload our tab, so this is a one way flag. The stop is something that'll
+     * only get triggered when we're actually running on a Chromecast since it
+     * always happens in response to a message handler.
+     */
+    haveStarted = false;
+    /**
+     * Cached result of the isChromecast test.
+     */
+    isChromecast: boolean | undefined;
+    /**
+     * A callback to invoke to get the pairing code when we get a new incoming
+     * pairing request.
+     */
+    pairingCode: (() => string | undefined) | undefined;
+    /**
+     * A callback to invoke to get the ID of the collection that is currently
+     * being shown (if any).
+     */
+    collectionID: (() => string | undefined) | undefined;
+}
+
+/** Singleton instance of {@link CastReceiver}. */
+const castReceiver = new CastReceiver();
+
+/**
+ * Listen for incoming messages on the given {@link cast} receiver, replying to
+ * each of them with a pairing code obtained using the given {@link pairingCode}
+ * callback. Phase 2 of the pairing protocol.
+ *
+ * Calling this function multiple times is fine. The first time around, the
+ * Chromecast SDK will be loaded and will start listening. Subsequently, each
+ * time this is call, we'll update the callbacks, but otherwise just return
+ * immediately (letting the already attached listeners do their thing).
+ *
+ * @param pairingCode A callback to invoke to get the pairing code when we get a
+ * new incoming pairing request.
+ *
+ * @param collectionID A callback to invoke to get the ID of the collection that
+ * is currently being shown (if any).
+ *
+ * See: [Note: Pairing protocol].
+ */
+export const advertiseOnChromecast = (
+    pairingCode: () => string | undefined,
+    collectionID: () => string | undefined,
+) => {
+    // Always update the callbacks.
+    castReceiver.pairingCode = pairingCode;
+    castReceiver.collectionID = collectionID;
+
+    // No-op if we're already running.
+    if (castReceiver.haveStarted) return;
+
+    void loadingChromecastSDKIfNeeded().then((cast) => advertiseCode(cast));
+};
+
+/**
+ * Load the Chromecast Web Receiver SDK and return a reference to the `cast`
+ * global object that the SDK attaches to the window.
+ *
+ * Calling this function multiple times is fine, once the Chromecast SDK is
+ * loaded it'll thereafter return the reference to the same object always.
+ */
+const loadingChromecastSDKIfNeeded = async (): Promise<Cast> => {
+    if (castReceiver.cast) return castReceiver.cast;
+    if (castReceiver.loader) return await castReceiver.loader;
+
+    castReceiver.loader = new Promise((resolve) => {
+        const script = document.createElement("script");
+        script.src =
+            "https://www.gstatic.com/cast/sdk/libs/caf_receiver/v3/cast_receiver_framework.js";
+        script.addEventListener("load", () => {
+            castReceiver.cast = cast;
+            resolve(cast);
+        });
+        document.body.appendChild(script);
+    });
+
+    return await castReceiver.loader;
+};
+
+const advertiseCode = (cast: Cast) => {
+    if (castReceiver.haveStarted) {
+        // Multiple attempts raced to completion, ignore all but the first.
+        return;
+    }
+
+    castReceiver.haveStarted = true;
+
+    // Prepare the Chromecast "context".
+    const context = cast.framework.CastReceiverContext.getInstance();
+    const namespace = "urn:x-cast:pair-request";
+
+    const options = new cast.framework.CastReceiverOptions();
+    // We don't use the media features of the Cast SDK.
+    options.skipPlayersLoad = true;
+    // Do not stop the casting if the receiver is unreachable. A user should be
+    // able to start a cast on their phone and then put it away, leaving the
+    // cast running on their big screen.
+    options.disableIdleTimeout = true;
+
+    type ListenerProps = {
+        senderId: string;
+        data: unknown;
+    };
+
+    // Reply with the code that we have if anyone asks over Chromecast.
+    const incomingMessageListener = ({ senderId, data }: ListenerProps) => {
+        // The collection ID with is currently paired (if any).
+        const pairedCollectionID = castReceiver.collectionID?.();
+
+        // The collection ID in the request (if any).
+        const collectionID =
+            data &&
+            typeof data == "object" &&
+            typeof data["collectionID"] == "string"
+                ? data["collectionID"]
+                : undefined;
+
+        // If the request does not have a collectionID (or if we're not showing
+        // anything currently), forego this check.
+
+        if (collectionID && pairedCollectionID) {
+            // If we get another connection request for a _different_ collection
+            // ID, stop the app to allow the second device to reconnect using a
+            // freshly generated pairing code.
+            if (pairedCollectionID != collectionID) {
+                log.info(`request for a new collection ${collectionID}`);
+                context.stop();
+            } else {
+                // Duplicate request for same collection that we're already
+                // showing. Ignore.
+            }
+            return;
+        }
+
+        const code = castReceiver.pairingCode?.();
+        if (!code) {
+            // No code, but if we're already showing a collection, then ignore.
+            if (pairedCollectionID) return;
+
+            // Our caller waits until it has a pairing code before it calls
+            // `advertiseCode`, but there is still an edge case where we can
+            // find ourselves without a pairing code:
+            //
+            // 1. The current pairing code expires. We start the process to get
+            //    a new one.
+            //
+            // 2. But before that happens, someone connects.
+            //
+            // The window where this can happen is short, so if we do find
+            // ourselves in this scenario, just shutdown.
+            log.error("got pairing request when refreshing pairing codes");
+            context.stop();
+            return;
+        }
+
+        context.sendCustomMessage(namespace, senderId, { code });
+    };
+
+    context.addCustomMessageListener(
+        namespace,
+        // We need to cast, the `senderId` is present in the message we get but
+        // not present in the TypeScript type.
+        incomingMessageListener as unknown as SystemEventHandler,
+    );
+
+    // Close the (chromecast) tab if the sender disconnects.
+    //
+    // Chromecast does a "shutdown" of our cast app when we call `context.stop`.
+    // This translates into it closing the tab where it is showing our app.
+    context.addEventListener(
+        cast.framework.system.EventType.SENDER_DISCONNECTED,
+        () => context.stop(),
+    );
+
+    // Start listening for Chromecast connections.
+    context.start(options);
+};
+
+/**
+ * Return true if we're running on a Chromecast device.
+ *
+ * This allows changing our app's behaviour when we're running on Chromecast.
+ * Such checks are needed because during our testing we found that in practice,
+ * some processing is too heavy for Chromecast hardware (we tested with a 2nd
+ * gen device, this might not be true for newer variants).
+ *
+ * This variable is lazily updated when we enter {@link renderableImageURLs}. It
+ * is kept at the top level to avoid passing it around.
+ */
+export const isChromecast = () => {
+    let isCast = castReceiver.isChromecast;
+    if (isCast === undefined) {
+        isCast = window.navigator.userAgent.includes("CrKey");
+        castReceiver.isChromecast = isCast;
+    }
+    return isCast;
+};

web/apps/cast/src/services/detect-type.ts

@@ -9,6 +9,9 @@ import FileType from "file-type";
  *
  * It first peeks into the file's initial contents to detect the MIME type. If
  * that doesn't give any results, it tries to deduce it from the file's name.
+ *
+ * For the list of returned extensions, see (for our installed version):
+ * https://github.com/sindresorhus/file-type/blob/main/core.d.ts
  */
 export const detectMediaMIMEType = async (file: File): Promise<string> => {
     const chunkSizeForTypeDetection = 4100;

web/apps/cast/src/services/pair.ts

@@ -1,9 +1,8 @@
 import log from "@/next/log";
+import { wait } from "@/utils/promise";
 import { boxSealOpen, toB64 } from "@ente/shared/crypto/internal/libsodium";
 import castGateway from "@ente/shared/network/cast";
-import { wait } from "@ente/shared/utils";
 import _sodium from "libsodium-wrappers";
-import { type Cast } from "../utils/cast-receiver";
 
 export interface Registration {
     /** A pairing code shown on the screen. A client can use this to connect. */
@@ -100,105 +99,6 @@ export const register = async (): Promise<Registration> => {
     return { pairingCode, publicKeyB64, privateKeyB64 };
 };
 
-/**
- * Listen for incoming messages on the given {@link cast} receiver, replying to
- * each of them with a pairing code obtained using the given {@link pairingCode}
- * callback. Phase 2 of the pairing protocol.
- *
- * See: [Note: Pairing protocol].
- */
-export const advertiseCode = (
-    cast: Cast,
-    pairingCode: () => string | undefined,
-) => {
-    // Prepare the Chromecast "context".
-    const context = cast.framework.CastReceiverContext.getInstance();
-    const namespace = "urn:x-cast:pair-request";
-
-    const options = new cast.framework.CastReceiverOptions();
-    // We don't use the media features of the Cast SDK.
-    options.skipPlayersLoad = true;
-    // Do not stop the casting if the receiver is unreachable. A user should be
-    // able to start a cast on their phone and then put it away, leaving the
-    // cast running on their big screen.
-    options.disableIdleTimeout = true;
-
-    // The collection ID with which we paired. If we get another connection
-    // request for a different collection ID, restart the app to allow them to
-    // reconnect using a freshly generated pairing code.
-    //
-    // If the request does not have a collectionID, forego this check.
-    let pairedCollectionID: string | undefined;
-
-    type ListenerProps = {
-        senderId: string;
-        data: unknown;
-    };
-
-    // Reply with the code that we have if anyone asks over Chromecast.
-    const incomingMessageListener = ({ senderId, data }: ListenerProps) => {
-        const restart = (reason: string) => {
-            log.error(`Restarting app because ${reason}`);
-            // context.stop will close the tab but it'll get reopened again
-            // immediately since the client app will reconnect in the scenarios
-            // where we're calling this function.
-            context.stop();
-        };
-
-        const collectionID =
-            data &&
-            typeof data == "object" &&
-            typeof data["collectionID"] == "string"
-                ? data["collectionID"]
-                : undefined;
-
-        if (pairedCollectionID && pairedCollectionID != collectionID) {
-            restart(`incoming request for a new collection ${collectionID}`);
-            return;
-        }
-
-        pairedCollectionID = collectionID;
-
-        const code = pairingCode();
-        if (!code) {
-            // Our caller waits until it has a pairing code before it calls
-            // `advertiseCode`, but there is still an edge case where we can
-            // find ourselves without a pairing code:
-            //
-            // 1. The current pairing code expires. We start the process to get
-            //    a new one.
-            //
-            // 2. But before that happens, someone connects.
-            //
-            // The window where this can happen is short, so if we do find
-            // ourselves in this scenario,
-            restart("we got a pairing request when refreshing pairing codes");
-            return;
-        }
-
-        context.sendCustomMessage(namespace, senderId, { code });
-    };
-
-    context.addCustomMessageListener(
-        namespace,
-        // We need to cast, the `senderId` is present in the message we get but
-        // not present in the TypeScript type.
-        incomingMessageListener as unknown as SystemEventHandler,
-    );
-
-    // Close the (chromecast) tab if the sender disconnects.
-    //
-    // Chromecast does a "shutdown" of our cast app when we call `context.stop`.
-    // This translates into it closing the tab where it is showing our app.
-    context.addEventListener(
-        cast.framework.system.EventType.SENDER_DISCONNECTED,
-        () => context.stop(),
-    );
-
-    // Start listening for Chromecast connections.
-    context.start(options);
-};
-
 /**
  * Ask museum if anyone has sent a (encrypted) payload corresponding to the
  * given pairing code. If so, decrypt it using our private key and return the

web/apps/cast/src/services/cast.ts → web/apps/cast/src/services/render.ts

@@ -1,14 +1,23 @@
 import { FILE_TYPE } from "@/media/file-type";
-import { isNonWebImageFileExtension } from "@/media/formats";
+import { isHEICExtension, isNonWebImageFileExtension } from "@/media/formats";
 import { decodeLivePhoto } from "@/media/live-photo";
+import { createHEICConvertComlinkWorker } from "@/media/worker/heic-convert";
+import type { DedicatedHEICConvertWorker } from "@/media/worker/heic-convert.worker";
 import { nameAndExtension } from "@/next/file";
 import log from "@/next/log";
+import type { ComlinkWorker } from "@/next/worker/comlink-worker";
 import { shuffled } from "@/utils/array";
-import { ensure, ensureString } from "@/utils/ensure";
+import { wait } from "@/utils/promise";
 import ComlinkCryptoWorker from "@ente/shared/crypto";
+import { ApiError } from "@ente/shared/error";
 import HTTPService from "@ente/shared/network/HTTPService";
-import { getCastFileURL, getEndpoint } from "@ente/shared/network/api";
-import { wait } from "@ente/shared/utils";
+import {
+    getCastFileURL,
+    getCastThumbnailURL,
+    getEndpoint,
+} from "@ente/shared/network/api";
+import type { AxiosResponse } from "axios";
+import type { CastData } from "services/cast-data";
 import { detectMediaMIMEType } from "services/detect-type";
 import {
     EncryptedEnteFile,
@@ -16,53 +25,20 @@ import {
     FileMagicMetadata,
     FilePublicMagicMetadata,
 } from "types/file";
+import { isChromecast } from "./chromecast";
 
 /**
- * Save the data received after pairing with a sender into local storage.
- *
- * We will read in back when we start the slideshow.
+ * If we're using HEIC conversion, then this variable caches the comlink web
+ * worker we're using to perform the actual conversion.
  */
-export const storeCastData = (payload: unknown) => {
-    if (!payload || typeof payload != "object")
-        throw new Error("Unexpected cast data");
-
-    // Iterate through all the keys of the payload object and save them to
-    // localStorage. We don't validate here, we'll validate when we read these
-    // values back in `readCastData`.
-    for (const key in payload) {
-        window.localStorage.setItem(key, payload[key]);
-    }
-};
-
-interface CastData {
-    /** A key to decrypt the collection we are casting. */
-    collectionKey: string;
-    /** A credential to use for fetching media files for this cast session. */
-    castToken: string;
-}
-
-/**
- * Read back the cast data we got after pairing.
- *
- * Sibling of {@link storeCastData}. It throws an error if the expected data is
- * not present in localStorage.
- */
-export const readCastData = (): CastData => {
-    const collectionKey = ensureString(localStorage.getItem("collectionKey"));
-    const castToken = ensureString(localStorage.getItem("castToken"));
-    return { collectionKey, castToken };
-};
-
-type RenderableImageURLPair = [url: string, nextURL: string];
+let heicWorker: ComlinkWorker<typeof DedicatedHEICConvertWorker> | undefined;
 
 /**
  * An async generator function that loops through all the files in the
- * collection, returning renderable URLs to each that can be displayed in a
- * slideshow.
+ * collection, returning renderable image URLs to each that can be displayed in
+ * a slideshow.
  *
- * Each time it resolves with a pair of URLs (a {@link RenderableImageURLPair}),
- * one for the next slideshow image, and one for the slideshow image that will
- * be displayed after that. It also pre-fetches the next to next URL each time.
+ * Each time it resolves with a (data) URL for the slideshow image to show next.
  *
  * If there are no renderable image in the collection, the sequence ends by
  * yielding `{done: true}`.
@@ -73,14 +49,18 @@ type RenderableImageURLPair = [url: string, nextURL: string];
  *
  * The generator ignores errors in the fetching and decoding of individual
  * images in the collection, skipping the erroneous ones and moving onward to
- * the next one. It will however throw if there are errors when getting the
- * collection itself. This can happen both the first time, or when we are about
- * to loop around to the start of the collection.
+ * the next one.
+ *
+ * - It will however throw if there are errors when getting the collection
+ *   itself. This can happen both the first time, or when we are about to loop
+ *   around to the start of the collection.
+ *
+ * - It will also throw if three consecutive image fail.
  *
  * @param castData The collection to show and credentials to fetch the files
  * within it.
  */
-export const renderableImageURLs = async function* (castData: CastData) {
+export const imageURLGenerator = async function* (castData: CastData) {
     const { collectionKey, castToken } = castData;
 
     /**
@@ -89,11 +69,8 @@ export const renderableImageURLs = async function* (castData: CastData) {
      */
     const previousURLs: string[] = [];
 
-    /** The URL pair that we will yield */
-    const urls: string[] = [];
-
     /** Number of milliseconds to keep the slide on the screen. */
-    const slideDuration = 10000; /* 10 s */
+    const slideDuration = 12000; /* 12 s */
 
     /**
      * Time when we last yielded.
@@ -108,6 +85,14 @@ export const renderableImageURLs = async function* (castData: CastData) {
     // bit, for the user to see the checkmark animation as reassurance).
     lastYieldTime -= slideDuration - 2500; /* wait at most 2.5 s */
 
+    /**
+     * Number of time we have caught an exception while trying to generate an
+     * image URL for individual files.
+     *
+     * When this happens three times consecutively, we throw.
+     */
+    let consecutiveFailures = 0;
+
     while (true) {
         const encryptedFiles = shuffled(
             await getEncryptedCollectionFiles(castToken),
@@ -118,30 +103,34 @@ export const renderableImageURLs = async function* (castData: CastData) {
         for (const encryptedFile of encryptedFiles) {
             const file = await decryptEnteFile(encryptedFile, collectionKey);
 
-            if (!isFileEligibleForCast(file)) continue;
+            if (!isFileEligible(file)) continue;
 
-            console.log("will start createRenderableURL", new Date());
+            let url: string;
             try {
-                urls.push(await createRenderableURL(castToken, file));
+                url = await createRenderableURL(castToken, file);
+                consecutiveFailures = 0;
                 haveEligibleFiles = true;
             } catch (e) {
+                consecutiveFailures += 1;
+                // 1, 2, bang!
+                if (consecutiveFailures == 3) throw e;
+
+                if (e instanceof ApiError && e.httpStatusCode == 401) {
+                    // The token has expired. This can happen, e.g., if the user
+                    // opens the dialog to cast again, causing the client to
+                    // invalidate existing tokens.
+                    //
+                    //  Rethrow the error, which will bring us back to the
+                    // pairing page.
+                    throw e;
+                }
+
+                // On all other errors (including temporary network issues),
                 log.error("Skipping unrenderable file", e);
+                await wait(100); /* Breathe */
                 continue;
             }
 
-            console.log("did end createRenderableURL", new Date());
-
-            // Need at least a pair.
-            //
-            // There are two scenarios:
-            //
-            // - First run: urls will initially be empty, so gobble two.
-            //
-            // - Subsequently, urls will have the "next" / "preloaded" URL left
-            //   over from the last time. We'll promote that to being the one
-            //   that'll get displayed, and preload another one.
-            // if (urls.length < 2) continue;
-
             // The last element of previousURLs is the URL that is currently
             // being shown on screen.
             //
@@ -150,23 +139,14 @@ export const renderableImageURLs = async function* (castData: CastData) {
             if (previousURLs.length > 1)
                 URL.revokeObjectURL(previousURLs.shift());
 
-            // The URL that'll now get displayed on screen.
-            const url = ensure(urls.shift());
-            // The URL that we're preloading for next time around.
-            const nextURL = ""; //ensure(urls[0]);
-
             previousURLs.push(url);
 
-            const urlPair: RenderableImageURLPair = [url, nextURL];
-
             const elapsedTime = Date.now() - lastYieldTime;
-            if (elapsedTime > 0 && elapsedTime < slideDuration) {
-                console.log("waiting", slideDuration - elapsedTime);
+            if (elapsedTime > 0 && elapsedTime < slideDuration)
                 await wait(slideDuration - elapsedTime);
-            }
 
             lastYieldTime = Date.now();
-            yield urlPair;
+            yield url;
         }
 
         // This collection does not have any files that we can show.
@@ -185,7 +165,7 @@ const getEncryptedCollectionFiles = async (
 ): Promise<EncryptedEnteFile[]> => {
     let files: EncryptedEnteFile[] = [];
     let sinceTime = 0;
-    let resp;
+    let resp: AxiosResponse;
     do {
         resp = await HTTPService.get(
             `${getEndpoint()}/cast/diff`,
@@ -269,12 +249,19 @@ const decryptEnteFile = async (
     return file;
 };
 
-const isFileEligibleForCast = (file: EnteFile) => {
+const isFileEligible = (file: EnteFile) => {
     if (!isImageOrLivePhoto(file)) return false;
     if (file.info.fileSize > 100 * 1024 * 1024) return false;
 
+    // This check is fast but potentially incorrect because in practice we do
+    // encounter files that are incorrectly named and have a misleading
+    // extension. To detect the actual type, we need to sniff the MIME type, but
+    // that requires downloading and decrypting the file first.
     const [, extension] = nameAndExtension(file.metadata.title);
-    if (isNonWebImageFileExtension(extension)) return false;
+    if (isNonWebImageFileExtension(extension)) {
+        // Of the known non-web types, we support HEIC.
+        return isHEICExtension(extension);
+    }
 
     return true;
 };
@@ -284,6 +271,12 @@ const isImageOrLivePhoto = (file: EnteFile) => {
     return fileType == FILE_TYPE.IMAGE || fileType == FILE_TYPE.LIVE_PHOTO;
 };
 
+export const heicToJPEG = async (heicBlob: Blob) => {
+    let worker = heicWorker;
+    if (!worker) heicWorker = worker = createHEICConvertComlinkWorker();
+    return await (await worker.remote).heicToJPEG(heicBlob);
+};
+
 /**
  * Create and return a new data URL that can be used to show the given
  * {@link file} in our slideshow image viewer.
@@ -291,29 +284,50 @@ const isImageOrLivePhoto = (file: EnteFile) => {
  * Once we're done showing the file, the URL should be revoked using
  * {@link URL.revokeObjectURL} to free up browser resources.
  */
-const createRenderableURL = async (castToken: string, file: EnteFile) =>
-    URL.createObjectURL(await renderableImageBlob(castToken, file));
+const createRenderableURL = async (castToken: string, file: EnteFile) => {
+    const imageBlob = await renderableImageBlob(castToken, file);
+    return URL.createObjectURL(imageBlob);
+};
 
 const renderableImageBlob = async (castToken: string, file: EnteFile) => {
-    const fileName = file.metadata.title;
-    let blob = await downloadFile(castToken, file);
-    if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
-        const { imageData } = await decodeLivePhoto(fileName, blob);
+    const shouldUseThumbnail = isChromecast();
+
+    let blob = await downloadFile(castToken, file, shouldUseThumbnail);
+
+    let fileName = file.metadata.title;
+    if (!shouldUseThumbnail && file.metadata.fileType == FILE_TYPE.LIVE_PHOTO) {
+        const { imageData, imageFileName } = await decodeLivePhoto(
+            fileName,
+            blob,
+        );
+        fileName = imageFileName;
         blob = new Blob([imageData]);
     }
+
+    // We cannot rely on the file's extension to detect the file type, some
+    // files are incorrectly named. So use a MIME type sniffer first, but if
+    // that fails than fallback to the extension.
     const mimeType = await detectMediaMIMEType(new File([blob], fileName));
     if (!mimeType)
         throw new Error(`Could not detect MIME type for file ${fileName}`);
+
+    if (mimeType == "image/heif" || mimeType == "image/heic")
+        blob = await heicToJPEG(blob);
+
     return new Blob([blob], { type: mimeType });
 };
 
-const downloadFile = async (castToken: string, file: EnteFile) => {
+const downloadFile = async (
+    castToken: string,
+    file: EnteFile,
+    shouldUseThumbnail: boolean,
+) => {
     if (!isImageOrLivePhoto(file))
         throw new Error("Can only cast images and live photos");
 
-    const url = getCastFileURL(file.id);
-    // TODO(MR): Remove if usused eventually
-    // const url = getCastThumbnailURL(file.id);
+    const url = shouldUseThumbnail
+        ? getCastThumbnailURL(file.id)
+        : getCastFileURL(file.id);
     const resp = await HTTPService.get(
         url,
         null,
@@ -327,9 +341,11 @@ const downloadFile = async (castToken: string, file: EnteFile) => {
     const cryptoWorker = await ComlinkCryptoWorker.getInstance();
     const decrypted = await cryptoWorker.decryptFile(
         new Uint8Array(resp.data),
-        await cryptoWorker.fromB64(file.file.decryptionHeader),
-        // TODO(MR): Remove if usused eventually
-        // await cryptoWorker.fromB64(file.thumbnail.decryptionHeader),
+        await cryptoWorker.fromB64(
+            shouldUseThumbnail
+                ? file.thumbnail.decryptionHeader
+                : file.file.decryptionHeader,
+        ),
         file.key,
     );
     return new Response(decrypted).blob();

web/apps/cast/src/utils/cast-receiver.tsx

@@ -1,32 +0,0 @@
-/// <reference types="chromecast-caf-receiver" />
-
-export type Cast = typeof cast;
-
-let _cast: Cast | undefined;
-let _loader: Promise<Cast> | undefined;
-
-/**
- * Load the Chromecast Web Receiver SDK and return a reference to the `cast`
- * global object that the SDK attaches to the window.
- *
- * Calling this function multiple times is fine, once the Chromecast SDK is
- * loaded it'll thereafter return the reference to the same object always.
- *
- * https://developers.google.com/cast/docs/web_receiver/basic
- */
-export const castReceiverLoadingIfNeeded = async (): Promise<Cast> => {
-    if (_cast) return _cast;
-    if (_loader) return await _loader;
-
-    _loader = new Promise((resolve) => {
-        const script = document.createElement("script");
-        script.src =
-            "https://www.gstatic.com/cast/sdk/libs/caf_receiver/v3/cast_receiver_framework.js";
-
-        script.addEventListener("load", () => resolve(cast));
-        document.body.appendChild(script);
-    });
-    const c = await _loader;
-    _cast = c;
-    return c;
-};

web/apps/photos/package.json

@@ -23,7 +23,6 @@
         "ffmpeg-wasm": "file:./thirdparty/ffmpeg-wasm",
         "formik": "^2.1.5",
         "hdbscan": "0.0.1-alpha.5",
-        "heic-convert": "^2.0.0",
         "idb": "^7.1.1",
         "leaflet": "^1.9.4",
         "leaflet-defaulticon-compatibility": "^0.1.1",

web/apps/photos/src/components/Collections/CollectionOptions/AlbumCastDialog.tsx

@@ -32,7 +32,11 @@ declare global {
     }
 }
 
-export default function AlbumCastDialog(props: Props) {
+export default function AlbumCastDialog({
+    show,
+    onHide,
+    currentCollection,
+}: Props) {
     const [view, setView] = useState<
         "choose" | "auto" | "pin" | "auto-cast-error"
     >("choose");
@@ -51,7 +55,7 @@ export default function AlbumCastDialog(props: Props) {
     ) => {
         try {
             await doCast(value.trim());
-            props.onHide();
+            onHide();
         } catch (e) {
             const error = e as Error;
             let fieldError: string;
@@ -80,8 +84,8 @@ export default function AlbumCastDialog(props: Props) {
         // ok, they exist. let's give them the good stuff.
         const payload = JSON.stringify({
             castToken: castToken,
-            collectionID: props.currentCollection.id,
-            collectionKey: props.currentCollection.key,
+            collectionID: currentCollection.id,
+            collectionKey: currentCollection.key,
         });
         const encryptedPayload = await boxSeal(btoa(payload), tvPublicKeyB64);
 
@@ -89,7 +93,7 @@ export default function AlbumCastDialog(props: Props) {
         await castGateway.publishCastPayload(
             pin,
             encryptedPayload,
-            props.currentCollection.id,
+            currentCollection.id,
             castToken,
         );
     };
@@ -119,7 +123,7 @@ export default function AlbumCastDialog(props: Props) {
                             doCast(code)
                                 .then(() => {
                                     setView("choose");
-                                    props.onHide();
+                                    onHide();
                                 })
                                 .catch((e) => {
                                     setView("auto-cast-error");
@@ -129,8 +133,9 @@ export default function AlbumCastDialog(props: Props) {
                     },
                 );
 
+                const collectionID = currentCollection.id;
                 session
-                    .sendMessage("urn:x-cast:pair-request", {})
+                    .sendMessage("urn:x-cast:pair-request", { collectionID })
                     .then(() => {
                         log.debug(() => "Message sent successfully");
                     })
@@ -142,16 +147,16 @@ export default function AlbumCastDialog(props: Props) {
     }, [view]);
 
     useEffect(() => {
-        if (props.show) {
+        if (show) {
             castGateway.revokeAllTokens();
         }
-    }, [props.show]);
+    }, [show]);
 
     return (
         <DialogBoxV2
             sx={{ zIndex: 1600 }}
-            open={props.show}
-            onClose={props.onHide}
+            open={show}
+            onClose={onHide}
             attributes={{
                 title: t("CAST_ALBUM_TO_TV"),
             }}

web/apps/photos/src/components/Collections/CollectionOptions/SharedCollectionOption.tsx

@@ -1,6 +1,7 @@
 import { OverflowMenuOption } from "@ente/shared/components/OverflowMenu/option";
 import ArchiveOutlined from "@mui/icons-material/ArchiveOutlined";
 import LogoutIcon from "@mui/icons-material/Logout";
+import TvIcon from "@mui/icons-material/Tv";
 import Unarchive from "@mui/icons-material/Unarchive";
 import { t } from "i18next";
 import { CollectionActions } from ".";
@@ -45,6 +46,15 @@ export function SharedCollectionOption({
             >
                 {t("LEAVE_ALBUM")}
             </OverflowMenuOption>
+            <OverflowMenuOption
+                startIcon={<TvIcon />}
+                onClick={handleCollectionAction(
+                    CollectionActions.SHOW_ALBUM_CAST_DIALOG,
+                    false,
+                )}
+            >
+                {t("CAST_ALBUM_TO_TV")}
+            </OverflowMenuOption>
         </>
     );
 }

web/apps/photos/src/services/export/index.ts

@@ -3,12 +3,12 @@ import { decodeLivePhoto } from "@/media/live-photo";
 import type { Metadata } from "@/media/types/file";
 import { ensureElectron } from "@/next/electron";
 import log from "@/next/log";
+import { wait } from "@/utils/promise";
 import { CustomError } from "@ente/shared/error";
 import { Events, eventBus } from "@ente/shared/events";
 import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage";
 import { formatDateTimeShort } from "@ente/shared/time/format";
 import { User } from "@ente/shared/user/types";
-import { wait } from "@ente/shared/utils";
 import QueueProcessor, {
     CancellationStatus,
     RequestCanceller,

web/apps/photos/src/services/export/migration.ts

@@ -3,9 +3,9 @@ import { decodeLivePhoto } from "@/media/live-photo";
 import { ensureElectron } from "@/next/electron";
 import { nameAndExtension } from "@/next/file";
 import log from "@/next/log";
+import { wait } from "@/utils/promise";
 import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
 import { User } from "@ente/shared/user/types";
-import { wait } from "@ente/shared/utils";
 import { getLocalCollections } from "services/collectionService";
 import downloadManager from "services/download";
 import { getAllLocalFiles } from "services/fileService";

web/apps/photos/src/services/heic-convert.ts

@@ -1,9 +1,10 @@
+import { createHEICConvertComlinkWorker } from "@/media/worker/heic-convert";
+import type { DedicatedHEICConvertWorker } from "@/media/worker/heic-convert.worker";
 import log from "@/next/log";
 import { ComlinkWorker } from "@/next/worker/comlink-worker";
 import { CustomError } from "@ente/shared/error";
 import { retryAsyncFunction } from "@ente/shared/utils";
 import QueueProcessor from "@ente/shared/utils/queueProcessor";
-import { type DedicatedHEICConvertWorker } from "worker/heic-convert.worker";
 
 /**
  * Convert a HEIC image to a JPEG.
@@ -29,7 +30,7 @@ class HEICConverter {
         if (this.workerPool.length > 0) return;
         this.workerPool = [];
         for (let i = 0; i < WORKER_POOL_SIZE; i++)
-            this.workerPool.push(createComlinkWorker());
+            this.workerPool.push(createHEICConvertComlinkWorker());
     }
 
     async convert(fileBlob: Blob): Promise<Blob> {
@@ -79,7 +80,7 @@ class HEICConverter {
                 } catch (e) {
                     log.error("HEIC conversion failed", e);
                     convertWorker.terminate();
-                    this.workerPool.push(createComlinkWorker());
+                    this.workerPool.push(createHEICConvertComlinkWorker());
                     throw e;
                 }
             }, WAIT_TIME_BEFORE_NEXT_ATTEMPT_IN_MICROSECONDS),
@@ -99,9 +100,3 @@ class HEICConverter {
 
 /** The singleton instance of {@link HEICConverter}. */
 const converter = new HEICConverter();
-
-const createComlinkWorker = () =>
-    new ComlinkWorker<typeof DedicatedHEICConvertWorker>(
-        "heic-convert-worker",
-        new Worker(new URL("worker/heic-convert.worker.ts", import.meta.url)),
-    );

web/apps/photos/src/services/upload/thumbnail.ts

@@ -1,7 +1,9 @@
 import { FILE_TYPE, type FileTypeInfo } from "@/media/file-type";
+import { scaledImageDimensions } from "@/media/image";
 import log from "@/next/log";
 import { type Electron } from "@/next/types/ipc";
-import { withTimeout } from "@ente/shared/utils";
+import { ensure } from "@/utils/ensure";
+import { withTimeout } from "@/utils/promise";
 import * as ffmpeg from "services/ffmpeg";
 import { heicToJPEG } from "services/heic-convert";
 import { toDataOrPathOrZipEntry, type DesktopUploadItem } from "./types";
@@ -30,10 +32,10 @@ export const generateThumbnailWeb = async (
     fileTypeInfo: FileTypeInfo,
 ): Promise<Uint8Array> =>
     fileTypeInfo.fileType === FILE_TYPE.IMAGE
-        ? await generateImageThumbnailUsingCanvas(blob, fileTypeInfo)
+        ? await generateImageThumbnailWeb(blob, fileTypeInfo)
         : await generateVideoThumbnailWeb(blob);
 
-const generateImageThumbnailUsingCanvas = async (
+const generateImageThumbnailWeb = async (
     blob: Blob,
     { extension }: FileTypeInfo,
 ) => {
@@ -42,8 +44,12 @@ const generateImageThumbnailUsingCanvas = async (
         blob = await heicToJPEG(blob);
     }
 
+    return generateImageThumbnailUsingCanvas(blob);
+};
+
+const generateImageThumbnailUsingCanvas = async (blob: Blob) => {
     const canvas = document.createElement("canvas");
-    const canvasCtx = canvas.getContext("2d");
+    const canvasCtx = ensure(canvas.getContext("2d"));
 
     const imageURL = URL.createObjectURL(blob);
     await withTimeout(
@@ -53,7 +59,7 @@ const generateImageThumbnailUsingCanvas = async (
             image.onload = () => {
                 try {
                     URL.revokeObjectURL(imageURL);
-                    const { width, height } = scaledThumbnailDimensions(
+                    const { width, height } = scaledImageDimensions(
                         image.width,
                         image.height,
                         maxThumbnailDimension,
@@ -62,7 +68,7 @@ const generateImageThumbnailUsingCanvas = async (
                     canvas.height = height;
                     canvasCtx.drawImage(image, 0, 0, width, height);
                     resolve(undefined);
-                } catch (e) {
+                } catch (e: unknown) {
                     reject(e);
                 }
             };
@@ -73,6 +79,32 @@ const generateImageThumbnailUsingCanvas = async (
     return await compressedJPEGData(canvas);
 };
 
+const compressedJPEGData = async (canvas: HTMLCanvasElement) => {
+    let blob: Blob | undefined | null;
+    let prevSize = Number.MAX_SAFE_INTEGER;
+    let quality = 0.7;
+
+    do {
+        if (blob) prevSize = blob.size;
+        blob = await new Promise((resolve) => {
+            canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality);
+        });
+        quality -= 0.1;
+    } while (
+        quality >= 0.5 &&
+        blob &&
+        blob.size > maxThumbnailSize &&
+        percentageSizeDiff(blob.size, prevSize) >= 10
+    );
+
+    return new Uint8Array(await ensure(blob).arrayBuffer());
+};
+
+const percentageSizeDiff = (
+    newThumbnailSize: number,
+    oldThumbnailSize: number,
+) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize;
+
 const generateVideoThumbnailWeb = async (blob: Blob) => {
     try {
         return await ffmpeg.generateVideoThumbnailWeb(blob);
@@ -85,9 +117,9 @@ const generateVideoThumbnailWeb = async (blob: Blob) => {
     }
 };
 
-const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
+export const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
     const canvas = document.createElement("canvas");
-    const canvasCtx = canvas.getContext("2d");
+    const canvasCtx = ensure(canvas.getContext("2d"));
 
     const videoURL = URL.createObjectURL(blob);
     await withTimeout(
@@ -98,7 +130,7 @@ const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
             video.addEventListener("loadeddata", () => {
                 try {
                     URL.revokeObjectURL(videoURL);
-                    const { width, height } = scaledThumbnailDimensions(
+                    const { width, height } = scaledImageDimensions(
                         video.videoWidth,
                         video.videoHeight,
                         maxThumbnailDimension,
@@ -118,59 +150,6 @@ const generateVideoThumbnailUsingCanvas = async (blob: Blob) => {
     return await compressedJPEGData(canvas);
 };
 
-/**
- * Compute the size of the thumbnail to create for an image with the given
- * {@link width} and {@link height}.
- *
- * This function calculates a new size of an image for limiting it to maximum
- * width and height (both specified by {@link maxDimension}), while maintaining
- * aspect ratio.
- *
- * It returns `{0, 0}` for invalid inputs.
- */
-const scaledThumbnailDimensions = (
-    width: number,
-    height: number,
-    maxDimension: number,
-): { width: number; height: number } => {
-    if (width === 0 || height === 0) return { width: 0, height: 0 };
-    const widthScaleFactor = maxDimension / width;
-    const heightScaleFactor = maxDimension / height;
-    const scaleFactor = Math.min(widthScaleFactor, heightScaleFactor);
-    const thumbnailDimensions = {
-        width: Math.round(width * scaleFactor),
-        height: Math.round(height * scaleFactor),
-    };
-    if (thumbnailDimensions.width === 0 || thumbnailDimensions.height === 0)
-        return { width: 0, height: 0 };
-    return thumbnailDimensions;
-};
-
-const compressedJPEGData = async (canvas: HTMLCanvasElement) => {
-    let blob: Blob;
-    let prevSize = Number.MAX_SAFE_INTEGER;
-    let quality = 0.7;
-
-    do {
-        if (blob) prevSize = blob.size;
-        blob = await new Promise((resolve) => {
-            canvas.toBlob((blob) => resolve(blob), "image/jpeg", quality);
-        });
-        quality -= 0.1;
-    } while (
-        quality >= 0.5 &&
-        blob.size > maxThumbnailSize &&
-        percentageSizeDiff(blob.size, prevSize) >= 10
-    );
-
-    return new Uint8Array(await blob.arrayBuffer());
-};
-
-const percentageSizeDiff = (
-    newThumbnailSize: number,
-    oldThumbnailSize: number,
-) => ((oldThumbnailSize - newThumbnailSize) * 100) / oldThumbnailSize;
-
 /**
  * Generate a JPEG thumbnail for the given file or path using native tools.
  *

web/apps/photos/src/services/upload/uploadHttpClient.ts

@@ -1,9 +1,9 @@
 import log from "@/next/log";
+import { wait } from "@/utils/promise";
 import { CustomError, handleUploadError } from "@ente/shared/error";
 import HTTPService from "@ente/shared/network/HTTPService";
 import { getEndpoint, getUploadEndpoint } from "@ente/shared/network/api";
 import { getToken } from "@ente/shared/storage/localStorage/helpers";
-import { wait } from "@ente/shared/utils";
 import { EnteFile } from "types/file";
 import { MultipartUploadURLs, UploadFile, UploadURL } from "./uploadService";
 

web/apps/photos/src/services/upload/uploadManager.ts

@@ -6,11 +6,11 @@ import log from "@/next/log";
 import type { Electron } from "@/next/types/ipc";
 import { ComlinkWorker } from "@/next/worker/comlink-worker";
 import { ensure } from "@/utils/ensure";
+import { wait } from "@/utils/promise";
 import { getDedicatedCryptoWorker } from "@ente/shared/crypto";
 import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
 import { CustomError } from "@ente/shared/error";
 import { Events, eventBus } from "@ente/shared/events";
-import { wait } from "@ente/shared/utils";
 import { Canceler } from "axios";
 import { Remote } from "comlink";
 import {

web/apps/photos/src/utils/file/index.ts

@@ -5,10 +5,11 @@ import { lowercaseExtension } from "@/next/file";
 import log from "@/next/log";
 import { CustomErrorMessage, type Electron } from "@/next/types/ipc";
 import { workerBridge } from "@/next/worker/worker-bridge";
+import { withTimeout } from "@/utils/promise";
 import ComlinkCryptoWorker from "@ente/shared/crypto";
 import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
 import { User } from "@ente/shared/user/types";
-import { downloadUsingAnchor, withTimeout } from "@ente/shared/utils";
+import { downloadUsingAnchor } from "@ente/shared/utils";
 import { t } from "i18next";
 import isElectron from "is-electron";
 import { moveToHiddenCollection } from "services/collectionService";

web/docs/dependencies.md

@@ -141,6 +141,14 @@ some cases.
     became ESM only - for our limited use case, the custom Webpack configuration
     that entails is not worth the upgrade.
 
+-   [heic-convert](https://github.com/catdad-experiments/heic-convert) is used
+    for converting HEIC files (which browsers don't natively support) into JPEG.
+
+## Processing
+
+-   [comlink](https://github.com/GoogleChromeLabs/comlink) provides a minimal
+    layer on top of Web Workers to make them more easier to use.
+
 ## Photos app specific
 
 -   [react-dropzone](https://github.com/react-dropzone/react-dropzone/) is a

web/packages/accounts/components/ChangeEmail.tsx

@@ -1,3 +1,4 @@
+import { wait } from "@/utils/promise";
 import { changeEmail, sendOTTForEmailChange } from "@ente/accounts/api/user";
 import { APP_HOMES } from "@ente/shared/apps/constants";
 import { PageProps } from "@ente/shared/apps/types";
@@ -6,7 +7,6 @@ import FormPaperFooter from "@ente/shared/components/Form/FormPaper/Footer";
 import LinkButton from "@ente/shared/components/LinkButton";
 import SubmitButton from "@ente/shared/components/SubmitButton";
 import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage";
-import { wait } from "@ente/shared/utils";
 import { Alert, Box, TextField } from "@mui/material";
 import { Formik, FormikHelpers } from "formik";
 import { t } from "i18next";

web/packages/accounts/components/two-factor/VerifyForm.tsx

@@ -1,16 +1,15 @@
-import { Formik, FormikHelpers } from "formik";
-import { t } from "i18next";
-import { useRef, useState } from "react";
-import OtpInput from "react-otp-input";
-
+import { wait } from "@/utils/promise";
 import InvalidInputMessage from "@ente/accounts/components/two-factor/InvalidInputMessage";
 import {
     CenteredFlex,
     VerticallyCentered,
 } from "@ente/shared/components/Container";
 import SubmitButton from "@ente/shared/components/SubmitButton";
-import { wait } from "@ente/shared/utils";
 import { Box, Typography } from "@mui/material";
+import { Formik, FormikHelpers } from "formik";
+import { t } from "i18next";
+import { useRef, useState } from "react";
+import OtpInput from "react-otp-input";
 
 interface formValues {
     otp: string;

web/packages/media/formats.ts

@@ -24,3 +24,11 @@ const nonWebImageFileExtensions = [
  */
 export const isNonWebImageFileExtension = (extension: string) =>
     nonWebImageFileExtensions.includes(extension.toLowerCase());
+
+/**
+ * Return `true` if {@link extension} in for an HEIC-like file.
+ */
+export const isHEICExtension = (extension: string) => {
+    const ext = extension.toLowerCase();
+    return ext == "heic" || ext == "heif";
+};

web/packages/media/image.ts

@@ -0,0 +1,33 @@
+/**
+ * Compute optimal dimensions for a resized version of an image while
+ * maintaining aspect ratio of the source image.
+ *
+ * @param width The width of the source image.
+ *
+ * @param height The height of the source image.
+ *
+ * @param maxDimension The maximum width of height of the resized image.
+ *
+ * This function returns a new size limiting it to maximum width and height
+ * (both specified by {@link maxDimension}), while maintaining aspect ratio of
+ * the source {@link width} and {@link height}.
+ *
+ * It returns `{0, 0}` for invalid inputs.
+ */
+export const scaledImageDimensions = (
+    width: number,
+    height: number,
+    maxDimension: number,
+): { width: number; height: number } => {
+    if (width == 0 || height == 0) return { width: 0, height: 0 };
+    const widthScaleFactor = maxDimension / width;
+    const heightScaleFactor = maxDimension / height;
+    const scaleFactor = Math.min(widthScaleFactor, heightScaleFactor);
+    const resizedDimensions = {
+        width: Math.round(width * scaleFactor),
+        height: Math.round(height * scaleFactor),
+    };
+    if (resizedDimensions.width == 0 || resizedDimensions.height == 0)
+        return { width: 0, height: 0 };
+    return resizedDimensions;
+};

web/packages/media/package.json

@@ -5,6 +5,10 @@
     "dependencies": {
         "@/next": "*",
         "file-type": "16.5.4",
+        "heic-convert": "^2.1",
         "jszip": "^3.10"
+    },
+    "devDependencies": {
+        "@types/heic-convert": "^1.2.3"
     }
 }

web/packages/media/worker/heic-convert.ts

@@ -0,0 +1,11 @@
+import { ComlinkWorker } from "@/next/worker/comlink-worker";
+import type { DedicatedHEICConvertWorker } from "./heic-convert.worker";
+
+export const createHEICConvertWebWorker = () =>
+    new Worker(new URL("heic-convert.worker.ts", import.meta.url));
+
+export const createHEICConvertComlinkWorker = () =>
+    new ComlinkWorker<typeof DedicatedHEICConvertWorker>(
+        "heic-convert-worker",
+        createHEICConvertWebWorker(),
+    );

web/apps/photos/src/worker/heic-convert.worker.ts → web/packages/media/worker/heic-convert.worker.ts

@@ -7,7 +7,7 @@ export class DedicatedHEICConvertWorker {
     }
 }
 
-expose(DedicatedHEICConvertWorker, self);
+expose(DedicatedHEICConvertWorker);
 
 /**
  * Convert a HEIC file to a JPEG file.

web/packages/next/log.ts

@@ -3,6 +3,19 @@ import { isDevBuild } from "./env";
 import { logToDisk as webLogToDisk } from "./log-web";
 import { workerBridge } from "./worker/worker-bridge";
 
+/**
+ * Whether logs go to disk or are always emitted to the console.
+ */
+let shouldLogToDisk = true;
+
+/**
+ * By default, logs get saved into a ring buffer in the browser's local storage.
+ * However, in some contexts, e.g. when we're running as the cast app, there is
+ * no mechanism for the user to retrieve these logs. So this function exists as
+ * a way to disable the on disk logging and always use the console.
+ */
+export const disableDiskLogs = () => (shouldLogToDisk = false);
+
 /**
  * Write a {@link message} to the on-disk log.
  *
@@ -45,14 +58,14 @@ const messageWithError = (message: string, e?: unknown) => {
 
 const logError = (message: string, e?: unknown) => {
     const m = `[error] ${messageWithError(message, e)}`;
-    if (isDevBuild) console.error(m);
-    logToDisk(m);
+    console.error(m);
+    if (shouldLogToDisk) logToDisk(m);
 };
 
 const logWarn = (message: string, e?: unknown) => {
     const m = `[warn] ${messageWithError(message, e)}`;
-    if (isDevBuild) console.error(m);
-    logToDisk(m);
+    console.error(m);
+    if (shouldLogToDisk) logToDisk(m);
 };
 
 const logInfo = (...params: unknown[]) => {
@@ -60,8 +73,8 @@ const logInfo = (...params: unknown[]) => {
         .map((p) => (typeof p == "string" ? p : JSON.stringify(p)))
         .join(" ");
     const m = `[info] ${message}`;
-    if (isDevBuild) console.log(m);
-    logToDisk(m);
+    if (isDevBuild || !shouldLogToDisk) console.log(m);
+    if (shouldLogToDisk) logToDisk(m);
 };
 
 const logDebug = (param: () => unknown) => {
@@ -71,8 +84,8 @@ const logDebug = (param: () => unknown) => {
 /**
  * Ente's logger.
  *
- * This is an object that provides three functions to log at the corresponding
- * levels - error, info or debug.
+ * This is an object that provides functions to log at the corresponding levels:
+ * error, warn, info or debug.
  *
  * Whenever we need to save a log message to disk,
  *
@@ -89,8 +102,7 @@ export default {
      * any arbitrary object that we obtain, say, when in a try-catch handler (in
      * JavaScript any arbitrary value can be thrown).
      *
-     * The log is written to disk. In development builds, the log is also
-     * printed to the browser console.
+     * The log is written to disk and printed to the browser console.
      */
     error: logError,
     /**
@@ -104,8 +116,10 @@ export default {
      * This is meant as a replacement of {@link console.log}, and takes an
      * arbitrary number of arbitrary parameters that it then serializes.
      *
-     * The log is written to disk. In development builds, the log is also
-     * printed to the browser console.
+     * The log is written to disk. However, if logging to disk is disabled by
+     * using {@link disableDiskLogs}, then the log is printed to the console.
+     *
+     * In development builds, the log is always printed to the browser console.
      */
     info: logInfo,
     /**
@@ -118,8 +132,8 @@ export default {
      * The function can return an arbitrary value which is serialized before
      * being logged.
      *
-     * This log is NOT written to disk. And it is printed to the browser
-     * console, but only in development builds.
+     * This log is NOT written to disk. It is printed to the browser console,
+     * but only in development builds.
      */
     debug: logDebug,
 };

web/packages/shared/utils/index.ts

@@ -1,11 +1,4 @@
-/**
- * Wait for {@link ms} milliseconds
- *
- * This function is a promisified `setTimeout`. It returns a promise that
- * resolves after {@link ms} milliseconds.
- */
-export const wait = (ms: number) =>
-    new Promise((resolve) => setTimeout(resolve, ms));
+import { wait } from "@/utils/promise";
 
 export function downloadAsFile(filename: string, content: string) {
     const file = new Blob([content], {
@@ -52,23 +45,3 @@ export async function retryAsyncFunction<T>(
         }
     }
 }
-
-/**
- * Await the given {@link promise} for {@link timeoutMS} milliseconds. If it
- * does not resolve within {@link timeoutMS}, then reject with a timeout error.
- */
-export const withTimeout = async <T>(promise: Promise<T>, ms: number) => {
-    let timeoutId: ReturnType<typeof setTimeout>;
-    const rejectOnTimeout = new Promise<T>((_, reject) => {
-        timeoutId = setTimeout(
-            () => reject(new Error("Operation timed out")),
-            ms,
-        );
-    });
-    const promiseAndCancelTimeout = async () => {
-        const result = await promise;
-        clearTimeout(timeoutId);
-        return result;
-    };
-    return Promise.race([promiseAndCancelTimeout(), rejectOnTimeout]);
-};

web/packages/utils/promise.ts

@@ -0,0 +1,28 @@
+/**
+ * Wait for {@link ms} milliseconds
+ *
+ * This function is a promisified `setTimeout`. It returns a promise that
+ * resolves after {@link ms} milliseconds.
+ */
+export const wait = (ms: number) =>
+    new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Await the given {@link promise} for {@link timeoutMS} milliseconds. If it
+ * does not resolve within {@link timeoutMS}, then reject with a timeout error.
+ */
+export const withTimeout = async <T>(promise: Promise<T>, ms: number) => {
+    let timeoutId: ReturnType<typeof setTimeout>;
+    const rejectOnTimeout = new Promise<T>((_, reject) => {
+        timeoutId = setTimeout(
+            () => reject(new Error("Operation timed out")),
+            ms,
+        );
+    });
+    const promiseAndCancelTimeout = async () => {
+        const result = await promise;
+        clearTimeout(timeoutId);
+        return result;
+    };
+    return Promise.race([promiseAndCancelTimeout(), rejectOnTimeout]);
+};

web/yarn.lock

@@ -1015,6 +1015,11 @@
   resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-7946.0.14.tgz#319b63ad6df705ee2a65a73ef042c8271e696613"
   integrity sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==
 
+"@types/heic-convert@^1.2.3":
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/@types/heic-convert/-/heic-convert-1.2.3.tgz#0705f36e467e7b6180806edd0b3f1e673514ff8c"
+  integrity sha512-5LJ2fGuVk/gnOLihoT56xJwrXxfnNepGvrHwlW5ZtT3HS4jO1AqBaAHCxXUpnY9UaD3zYcyxXMRM2fNN1AFF/Q==
+
 "@types/hoist-non-react-statics@^3.3.1":
   version "3.3.5"
   resolved "https://registry.yarnpkg.com/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.5.tgz#dab7867ef789d87e2b4b0003c9d65c49cc44a494"
@@ -2839,7 +2844,7 @@ hdbscan@0.0.1-alpha.5:
   dependencies:
     kd-tree-javascript "^1.0.3"
 
-heic-convert@^2.0.0:
+heic-convert@^2.1:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/heic-convert/-/heic-convert-2.1.0.tgz#7f764529e37591ae263ef49582d1d0c13491526e"
   integrity sha512-1qDuRvEHifTVAj3pFIgkqGgJIr0M3X7cxEPjEp0oG4mo8GFjq99DpCo8Eg3kg17Cy0MTjxpFdoBHOatj7ZVKtg==
@@ -3321,7 +3326,7 @@ libsodium-wrappers@0.7.9:
   dependencies:
     libsodium "^0.7.0"
 
-libsodium@0.7.9, libsodium@^0.7.0:
+libsodium@^0.7.0:
   version "0.7.9"
   resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.9.tgz#4bb7bcbf662ddd920d8795c227ae25bbbfa3821b"
   integrity sha512-gfeADtR4D/CM0oRUviKBViMGXZDgnFdMKMzHsvBdqLBHd9ySi6EtYnmuhHVDDYgYpAO8eU8hEY+F8vIUAPh08A==