WIP

desktop/src/main/log.ts

@@ -104,11 +104,11 @@ export default {
      * function to call to get the log message instead of directly taking the
      * message. The provided function will only be called in development builds.
      *
-     * The function can return an arbitrary value which is serialied before
+     * The function can return an arbitrary value which is serialized before
      * being logged.
      *
-     * This log is not written to disk. It is printed to the main (Node.js)
-     * process console only on development builds.
+     * This log is NOT written to disk. And it is printed to the main (Node.js)
+     * process console, but only on development builds.
      */
     debug: logDebug,
 };

web/apps/auth/src/pages/_app.tsx

@@ -1,5 +1,6 @@
 import { CustomHead } from "@/next/components/Head";
 import { setupI18n } from "@/next/i18n";
+import { logStartupBanner } from "@/next/log-web";
 import {
     APPS,
     APP_TITLES,
@@ -16,15 +17,12 @@ import { MessageContainer } from "@ente/shared/components/MessageContainer";
 import AppNavbar from "@ente/shared/components/Navbar/app";
 import { PHOTOS_PAGES as PAGES } from "@ente/shared/constants/pages";
 import { useLocalState } from "@ente/shared/hooks/useLocalState";
-import {
-    clearLogsIfLocalStorageLimitExceeded,
-    logStartupBanner,
-} from "@ente/shared/logging/web";
 import HTTPService from "@ente/shared/network/HTTPService";
-import { LS_KEYS } from "@ente/shared/storage/localStorage";
+import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
 import { getTheme } from "@ente/shared/themes";
 import { THEME_COLOR } from "@ente/shared/themes/constants";
 import { SetTheme } from "@ente/shared/themes/types";
+import type { User } from "@ente/shared/user/types";
 import { CssBaseline, useMediaQuery } from "@mui/material";
 import { ThemeProvider } from "@mui/material/styles";
 import { t } from "i18next";
@@ -67,15 +65,12 @@ export default function App({ Component, pageProps }: AppProps) {
     );
 
     useEffect(() => {
-        //setup i18n
         setupI18n().finally(() => setIsI18nReady(true));
-        // set client package name in headers
+        const userId = (getData(LS_KEYS.USER) as User)?.id;
+        logStartupBanner(APPS.AUTH, userId);
         HTTPService.setHeaders({
             "X-Client-Package": CLIENT_PACKAGE_NAMES.get(APPS.AUTH),
         });
-        // setup logging
-        clearLogsIfLocalStorageLimitExceeded();
-        logStartupBanner(APPS.AUTH);
     }, []);
 
     const setUserOnline = () => setOffline(false);

web/apps/cast/src/services/readerService.ts

@@ -1,5 +1,5 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { logError } from "@ente/shared/sentry";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 
 export async function getUint8ArrayView(file: Blob): Promise<Uint8Array> {
     try {

web/apps/cast/src/services/typeDetectionService.ts

@@ -1,6 +1,6 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { CustomError } from "@ente/shared/error";
 import { logError } from "@ente/shared/sentry";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { FILE_TYPE } from "constants/file";
 import {
     KNOWN_NON_MEDIA_FORMATS,

web/apps/photos/src/components/PhotoList/dedupe.tsx

@@ -1,5 +1,5 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { FlexWrapper } from "@ente/shared/components/Container";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { Box, styled } from "@mui/material";
 import {
     DATE_CONTAINER_HEIGHT,

web/apps/photos/src/components/PhotoList/index.tsx

@@ -1,6 +1,6 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { FlexWrapper } from "@ente/shared/components/Container";
 import { formatDate, getDate, isSameDay } from "@ente/shared/time/format";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { Box, Checkbox, Link, Typography, styled } from "@mui/material";
 import {
     DATE_CONTAINER_HEIGHT,

web/apps/photos/src/components/Sidebar/DebugSection.tsx

@@ -4,8 +4,8 @@ import { useContext, useEffect, useState } from "react";
 import { Trans } from "react-i18next";
 
 import ElectronAPIs from "@/next/electron";
+import { savedLogs } from "@/next/log-web";
 import { addLogLine } from "@ente/shared/logging";
-import { getDebugLogs } from "@ente/shared/logging/web";
 import { downloadAsFile } from "@ente/shared/utils";
 import Typography from "@mui/material/Typography";
 import { EnteMenuItem } from "components/Menu/EnteMenuItem";
@@ -38,22 +38,17 @@ export default function DebugSection() {
             proceed: {
                 text: t("DOWNLOAD"),
                 variant: "accent",
-                action: downloadDebugLogs,
+                action: downloadLogs,
             },
             close: {
                 text: t("CANCEL"),
             },
         });
 
-    const downloadDebugLogs = () => {
-        addLogLine("exporting logs");
-        if (isElectron()) {
-            ElectronAPIs.openLogDirectory();
-        } else {
-            const logs = getDebugLogs();
-
-            downloadAsFile(`debug_logs_${Date.now()}.txt`, logs);
-        }
+    const downloadLogs = () => {
+        addLogLine("Downloading logs");
+        if (isElectron()) ElectronAPIs.openLogDirectory();
+        else downloadAsFile(`debug_logs_${Date.now()}.txt`, savedLogs());
     };
 
     return (

web/apps/photos/src/pages/_app.tsx

@@ -1,6 +1,7 @@
 import { CustomHead } from "@/next/components/Head";
 import ElectronAPIs from "@/next/electron";
 import { setupI18n } from "@/next/i18n";
+import { logStartupBanner } from "@/next/log-web";
 import { AppUpdateInfo } from "@/next/types/ipc";
 import {
     APPS,
@@ -26,10 +27,6 @@ import { CustomError } from "@ente/shared/error";
 import { Events, eventBus } from "@ente/shared/events";
 import { useLocalState } from "@ente/shared/hooks/useLocalState";
 import { addLogLine } from "@ente/shared/logging";
-import {
-    clearLogsIfLocalStorageLimitExceeded,
-    logStartupBanner,
-} from "@ente/shared/logging/web";
 import HTTPService from "@ente/shared/network/HTTPService";
 import { logError } from "@ente/shared/sentry";
 import { LS_KEYS, getData } from "@ente/shared/storage/localStorage";
@@ -41,6 +38,7 @@ import {
 import { getTheme } from "@ente/shared/themes";
 import { THEME_COLOR } from "@ente/shared/themes/constants";
 import { SetTheme } from "@ente/shared/themes/types";
+import type { User } from "@ente/shared/user/types";
 import ArrowForward from "@mui/icons-material/ArrowForward";
 import { CssBaseline, useMediaQuery } from "@mui/material";
 import { ThemeProvider } from "@mui/material/styles";
@@ -149,15 +147,12 @@ export default function App({ Component, pageProps }: AppProps) {
     );
 
     useEffect(() => {
-        //setup i18n
         setupI18n().finally(() => setIsI18nReady(true));
-        // set client package name in headers
+        const userId = (getData(LS_KEYS.USER) as User)?.id;
+        logStartupBanner(APPS.PHOTOS, userId);
         HTTPService.setHeaders({
             "X-Client-Package": CLIENT_PACKAGE_NAMES.get(APPS.PHOTOS),
         });
-        // setup logging
-        clearLogsIfLocalStorageLimitExceeded();
-        logStartupBanner(APPS.PHOTOS);
     }, []);
 
     useEffect(() => {

web/apps/photos/src/services/heic-convert/service.ts

@@ -1,9 +1,9 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { CustomError } from "@ente/shared/error";
 import { addLogLine } from "@ente/shared/logging";
 import { logError } from "@ente/shared/sentry";
 import { retryAsyncFunction } from "@ente/shared/utils";
 import QueueProcessor from "@ente/shared/utils/queueProcessor";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
 import { getDedicatedConvertWorker } from "utils/comlink/ComlinkConvertWorker";
 import { DedicatedConvertWorker } from "worker/convert.worker";

web/apps/photos/src/services/readerService.ts

@@ -1,5 +1,5 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { logError } from "@ente/shared/sentry";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { ElectronFile } from "types/upload";
 
 export async function getUint8ArrayView(

web/apps/photos/src/services/typeDetectionService.ts

@@ -1,6 +1,6 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { CustomError } from "@ente/shared/error";
 import { logError } from "@ente/shared/sentry";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { FILE_TYPE } from "constants/file";
 import {
     KNOWN_NON_MEDIA_FORMATS,

web/apps/photos/src/services/upload/thumbnailService.ts

@@ -1,9 +1,9 @@
 import ElectronAPIs from "@/next/electron";
+import { convertBytesToHumanReadable } from "@/next/file";
 import { CustomError } from "@ente/shared/error";
 import { addLogLine } from "@ente/shared/logging";
 import { getFileNameSize } from "@ente/shared/logging/web";
 import { logError } from "@ente/shared/sentry";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { FILE_TYPE } from "constants/file";
 import { BLACK_THUMBNAIL_BASE64 } from "constants/upload";
 import isElectron from "is-electron";

web/apps/photos/src/services/upload/uploader.ts

@@ -1,9 +1,9 @@
+import { convertBytesToHumanReadable } from "@/next/file";
 import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
 import { CustomError, handleUploadError } from "@ente/shared/error";
 import { addLocalLog, addLogLine } from "@ente/shared/logging";
 import { logError } from "@ente/shared/sentry";
 import { sleep } from "@ente/shared/utils";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import { Remote } from "comlink";
 import { MAX_FILE_SIZE_SUPPORTED, UPLOAD_RESULT } from "constants/upload";
 import { addToCollection } from "services/collectionService";

web/apps/photos/src/utils/file/index.ts

@@ -37,11 +37,11 @@ import {
 import { VISIBILITY_STATE } from "types/magicMetadata";
 import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata";
 
+import { convertBytesToHumanReadable } from "@/next/file";
 import ComlinkCryptoWorker from "@ente/shared/crypto";
 import { CustomError } from "@ente/shared/error";
 import { addLocalLog, addLogLine } from "@ente/shared/logging";
 import { isPlaybackPossible } from "@ente/shared/media/video-playback";
-import { convertBytesToHumanReadable } from "@ente/shared/utils/size";
 import isElectron from "is-electron";
 import { moveToHiddenCollection } from "services/collectionService";
 import {

web/packages/next/file.ts

@@ -1,3 +1,9 @@
+import type { ElectronFile } from "./types/file";
+
+export function getFileNameSize(file: File | ElectronFile) {
+    return `${file.name}_${convertBytesToHumanReadable(file.size)}`;
+}
+
 export function convertBytesToHumanReadable(
     bytes: number,
     precision = 2,

web/packages/next/log-web.ts

@@ -0,0 +1,81 @@
+import { isDevBuild } from "@/next/env";
+import { addLogLine } from "@ente/shared/logging";
+
+/**
+ * Log a standard startup banner.
+ *
+ * This helps us identify app starts and other environment details in the logs.
+ *
+ * @param appId An identifier of the app that is starting.
+ * @param userId The uid for the currently logged in user, if any.
+ */
+export const logStartupBanner = (appId: string, userId?: number) => {
+    // TODO (MR): Remove the need to lowercase it, change the enum itself.
+    const appIdL = appId.toLowerCase();
+    const sha = process.env.GIT_SHA;
+    const buildId = isDevBuild ? "dev " : sha ? `git ${sha} ` : "";
+
+    addLogLine(`Starting ente-${appIdL}-web ${buildId}uid ${userId ?? 0}`);
+};
+
+interface LogEntry {
+    timestamp: number;
+    logLine: string;
+}
+
+const lsKey = "logs";
+
+/**
+ * Record {@link message} in a persistent log storage.
+ *
+ * These strings, alongwith associated timestamps, get added to a small ring
+ * buffer, whose contents can be later be retrieved by using {@link savedLogs}.
+ *
+ * This ring buffer is persisted in the browser's local storage.
+ */
+export const persistLog = (message: string) => {
+    const maxCount = 1000;
+    const log: LogEntry = { logLine: message, timestamp: Date.now() };
+    try {
+        const logs = logEntries();
+        if (logs.length > maxCount) {
+            logs.slice(logs.length - maxCount);
+        }
+        logs.push(log);
+        localStorage.setItem(lsKey, JSON.stringify(logs));
+    } catch (e) {
+        console.error("Failed to persist log", e);
+        if (e instanceof Error && e.name === "QuotaExceededError") {
+            localStorage.removeItem(lsKey);
+        }
+    }
+};
+
+const logEntries = (): unknown[] => {
+    const s = localStorage.getItem("logs");
+    if (!s) return [];
+    const o: unknown = JSON.parse(s);
+    if (!(o && typeof o == "object" && "logs" in o && Array.isArray(o.logs))) {
+        console.error("Unexpected log entries obtained from local storage", o);
+        return [];
+    }
+    return o.logs;
+};
+
+/**
+ * Return a string containing all recently saved log messages.
+ *
+ * @see {@link persistLog}.
+ */
+export const savedLogs = () => logEntries().map(formatEntry).join("\n");
+
+const formatEntry = (e: unknown) => {
+    if (e && typeof e == "object" && "timestamp" in e && "logLine" in e) {
+        const timestamp = e.timestamp;
+        const logLine = e.logLine;
+        if (typeof timestamp == "number" && typeof logLine == "string") {
+            return `[${new Date(timestamp).toISOString()}] ${logLine}`;
+        }
+    }
+    return String(e);
+};

web/packages/next/log.ts

@@ -0,0 +1,104 @@
+import isElectron from "is-electron";
+import ElectronAPIs from "./electron";
+import { isDevBuild } from "./env";
+import { persistLog } from "./log-web";
+
+/**
+ * Write a {@link message} to the on-disk log.
+ *
+ * This is used by the renderer process (via the contextBridge) to add entries
+ * in the log that is saved on disk.
+ */
+export const logToDisk = (message: string) => {
+    if (isElectron()) ElectronAPIs.logToDisk(message);
+    else persistLog(message);
+};
+
+const logError = (message: string, e?: unknown) => {
+    if (!e) {
+        logError_(message);
+        return;
+    }
+
+    let es: string;
+    if (e instanceof Error) {
+        // In practice, we expect ourselves to be called with Error objects, so
+        // this is the happy path so to say.
+        es = `${e.name}: ${e.message}\n${e.stack}`;
+    } else {
+        // For the rest rare cases, use the default string serialization of e.
+        es = String(e);
+    }
+
+    logError_(`${message}: ${es}`);
+};
+
+const logError_ = (message: string) => {
+    const m = `[error] ${message}`;
+    if (isDevBuild) console.error(m);
+    logToDisk(m);
+};
+
+const logInfo = (...params: unknown[]) => {
+    const message = params
+        .map((p) => (typeof p == "string" ? p : JSON.stringify(p)))
+        .join(" ");
+    const m = `[info] ${message}`;
+    if (isDevBuild) console.log(m);
+    logToDisk(m);
+};
+
+const logDebug = (param: () => unknown) => {
+    if (isDevBuild) console.log("[debug]", param());
+};
+
+/**
+ * Ente's logger.
+ *
+ * This is an object that provides three functions to log at the corresponding
+ * levels - error, info or debug.
+ *
+ * Whenever we need to save a log message to disk,
+ *
+ * - When running under electron these messages are saved to the log maintained
+ *   by the electron app we're running under.
+ *
+ * - Otherwise such messages are written to a ring buffer in local storage.
+ */
+export default {
+    /**
+     * Log an error message with an optional associated error object.
+     *
+     * {@link e} is generally expected to be an `instanceof Error` but it can be
+     * any arbitrary object that we obtain, say, when in a try-catch handler (in
+     * JavaScript any arbitrary value can be thrown).
+     *
+     * The log is written to disk. In development builds, the log is also
+     * printed to the browser console.
+     */
+    error: logError,
+    /**
+     * Log a message.
+     *
+     * This is meant as a replacement of {@link console.log}, and takes an
+     * arbitrary number of arbitrary parameters that it then serializes.
+     *
+     * The log is written to disk. In development builds, the log is also
+     * printed to the browser console.
+     */
+    info: logInfo,
+    /**
+     * Log a debug message.
+     *
+     * To avoid running unnecessary code in release builds, this takes a
+     * function to call to get the log message instead of directly taking the
+     * message. The provided function will only be called in development builds.
+     *
+     * The function can return an arbitrary value which is serialized before
+     * being logged.
+     *
+     * This log is NOT written to disk. And it is printed to the browser
+     * console, but only in development builds.
+     */
+    debug: logDebug,
+};

web/packages/shared/logging/index.ts

@@ -1,9 +1,10 @@
+import ElectronAPIs from "@/next/electron";
 import { inWorker, isDevBuild } from "@/next/env";
+import log from "@/next/log";
+import { logWeb } from "@/next/web";
 import { logError } from "@ente/shared/sentry";
 import isElectron from "is-electron";
-import ElectronAPIs from "@/next/electron";
 import { workerBridge } from "../worker/worker-bridge";
-import { formatLog, logWeb } from "./web";
 
 export const MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB
 export const MAX_LOG_LINES = 1000;
@@ -45,13 +46,4 @@ export function addLogLine(
     }
 }
 
-export const addLocalLog = (getLog: () => string) => {
-    if (isDevBuild) {
-        console.log(
-            formatLog({
-                logLine: getLog(),
-                timestamp: Date.now(),
-            }),
-        );
-    }
-};
+export const addLocalLog = log.debug;

web/packages/shared/logging/web.ts

@@ -1,112 +0,0 @@
-import { isDevBuild } from "@/next/env";
-import { ElectronFile } from "@/next/types/file";
-import { logError } from "@ente/shared/sentry";
-import {
-    LS_KEYS,
-    getData,
-    removeData,
-    setData,
-} from "@ente/shared/storage/localStorage";
-import { addLogLine } from ".";
-import { formatDateTimeShort } from "../time/format";
-import type { User } from "../user/types";
-import { convertBytesToHumanReadable } from "../utils/size";
-
-export const MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB
-export const MAX_LOG_LINES = 1000;
-
-export interface Log {
-    timestamp: number;
-    logLine: string;
-}
-
-export function logWeb(logLine: string) {
-    try {
-        const log: Log = { logLine, timestamp: Date.now() };
-        const logs = getLogs();
-        if (logs.length > MAX_LOG_LINES) {
-            logs.slice(logs.length - MAX_LOG_LINES);
-        }
-        logs.push(log);
-        setLogs(logs);
-    } catch (e) {
-        if (e.name === "QuotaExceededError") {
-            deleteLogs();
-            logWeb("logs cleared");
-        }
-    }
-}
-
-export function getDebugLogs() {
-    return combineLogLines(getLogs());
-}
-
-export function getFileNameSize(file: File | ElectronFile) {
-    return `${file.name}_${convertBytesToHumanReadable(file.size)}`;
-}
-
-export const clearLogsIfLocalStorageLimitExceeded = () => {
-    try {
-        const logs = getDebugLogs();
-        const logSize = getStringSize(logs);
-        if (logSize > MAX_LOG_SIZE) {
-            deleteLogs();
-            logWeb("Logs cleared due to size limit exceeded");
-        } else {
-            try {
-                logWeb(`app started`);
-            } catch (e) {
-                deleteLogs();
-            }
-        }
-        logWeb(`logs size: ${convertBytesToHumanReadable(logSize)}`);
-    } catch (e) {
-        logError(
-            e,
-            "failed to clearLogsIfLocalStorageLimitExceeded",
-            undefined,
-            true,
-        );
-    }
-};
-
-/**
- * Log a standard startup banner.
- *
- * This helps us identify app starts and other environment details in the logs.
- *
- * @param appId An identifier of the app that is starting.
- */
-export const logStartupBanner = async (appId: string) => {
-    // TODO (MR): Remove the need to lowercase it, change the enum itself.
-    const appIdL = appId.toLowerCase();
-    const userID = (getData(LS_KEYS.USER) as User)?.id;
-    const sha = process.env.GIT_SHA;
-    const buildId = isDevBuild ? "dev " : sha ? `git ${sha} ` : "";
-
-    addLogLine(`Starting ente-${appIdL}-web ${buildId}uid ${userID}`);
-};
-
-function getLogs(): Log[] {
-    return getData(LS_KEYS.LOGS)?.logs ?? [];
-}
-
-function setLogs(logs: Log[]) {
-    setData(LS_KEYS.LOGS, { logs });
-}
-
-function deleteLogs() {
-    removeData(LS_KEYS.LOGS);
-}
-
-function getStringSize(str: string) {
-    return new Blob([str]).size;
-}
-
-export function formatLog(log: Log) {
-    return `[${formatDateTimeShort(log.timestamp)}] ${log.logLine}`;
-}
-
-function combineLogLines(logs: Log[]) {
-    return logs.map(formatLog).join("\n");
-}

web/packages/shared/storage/localStorage/index.ts

@@ -14,13 +14,13 @@ export enum LS_KEYS {
     EXPORT = "export",
     THUMBNAIL_FIX_STATE = "thumbnailFixState",
     LIVE_PHOTO_INFO_SHOWN_COUNT = "livePhotoInfoShownCount",
-    LOGS = "logs",
+    // LOGS = "logs",
     USER_DETAILS = "userDetails",
     COLLECTION_SORT_BY = "collectionSortBy",
     THEME = "theme",
     WAIT_TIME = "waitTime",
     API_ENDPOINT = "apiEndpoint",
-    // Moved to the new wrapper @/utils/local-storage
+    // Moved to the new wrapper @/next/local-storage
     // LOCALE = 'locale',
     MAP_ENABLED = "mapEnabled",
     SRP_SETUP_ATTRIBUTES = "srpSetupAttributes",

web/packages/shared/tsconfig.json

@@ -12,5 +12,11 @@
         "target": "es5",
         "useUnknownInCatchVariables": false
     },
-    "include": ["**/*.ts", "**/*.tsx", "**/*.js", "themes/mui-theme.d.ts"]
+    "include": [
+        "**/*.ts",
+        "**/*.tsx",
+        "**/*.js",
+        "themes/mui-theme.d.ts",
+        "../next/log-web.ts"
+    ]
 }