0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use the web native cache only - desktop side changes

Browse source
This commit is contained in:
Manav Rathi 2024-03-22 17:09:18 +05:30
parent 22e57669fb
commit 3dbf82552d
No known key found for this signature in database
12 changed files with 97 additions and 263 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
desktop/package.json
View file

@ -28,7 +28,6 @@
"electron-store": "^8.0.1",
"electron-updater": "^4.3.8",
"ffmpeg-static": "^5.1.0",
"get-folder-size": "^2.0.1",
"html-entities": "^2.4.0",
"jpeg-js": "^0.4.4",
"next-electron-server": "^1",
@ -38,7 +37,6 @@
"devDependencies": {
"@types/auto-launch": "^5.0.2",
"@types/ffmpeg-static": "^3.0.1",
"@types/get-folder-size": "^2.0.0",
"@typescript-eslint/eslint-plugin": "^7",
"@typescript-eslint/parser": "^7",
"concurrently": "^8",

37
desktop/src/api/cache.ts
View file

@ -1,37 +0,0 @@
import { ipcRenderer } from "electron/renderer";
import { existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import path from "path";
import { DiskCache } from "../services/diskCache";
const ENTE_CACHE_DIR_NAME = "ente";
const getCacheDirectory = async () => {
const defaultSystemCacheDir = await ipcRenderer.invoke("get-path", "cache");
return path.join(defaultSystemCacheDir, ENTE_CACHE_DIR_NAME);
};
const getCacheBucketDir = async (cacheName: string) => {
const cacheDir = await getCacheDirectory();
const cacheBucketDir = path.join(cacheDir, cacheName);
return cacheBucketDir;
};
export async function openDiskCache(
cacheName: string,
cacheLimitInBytes?: number,
) {
const cacheBucketDir = await getCacheBucketDir(cacheName);
await fs.mkdir(cacheBucketDir, { recursive: true });
return new DiskCache(cacheBucketDir, cacheLimitInBytes);
}
export async function deleteDiskCache(cacheName: string) {
const cacheBucketDir = await getCacheBucketDir(cacheName);
if (existsSync(cacheBucketDir)) {
await fs.rm(cacheBucketDir, { recursive: true, force: true });
return true;
} else {
return false;
}
}

97
desktop/src/main.ts
View file

@ -1,6 +1,21 @@
import { app, BrowserWindow } from "electron";
/**
* @file Entry point for the main (Node.js) process of our Electron app.
*
* The code in this file is invoked by Electron when our app starts -
* Conceptually (after all the transpilation etc has happened) this can be
* thought of `electron main.ts`. We're running in the context of the so called
* "main" process which runs in a Node.js environment.
*
* https://www.electronjs.org/docs/latest/tutorial/process-model#the-main-process
*/
import * as log from "electron-log";
import { app, BrowserWindow } from "electron/main";
import serveNextAt from "next-electron-server";
import { existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { initWatcher } from "./services/chokidar";
import { logErrorSentry } from "./services/sentry";
import { isDev } from "./utils/common";
import { addAllowOriginHeader } from "./utils/cors";
import { createWindow } from "./utils/createWindow";
@ -8,7 +23,6 @@ import { setupAppEventEmitter } from "./utils/events";
import setupIpcComs from "./utils/ipcComms";
import { setupLogging } from "./utils/logging";
import {
enableSharedArrayBufferSupport,
handleDockIconHideOnAutoLaunch,
handleDownloads,
handleExternalLinks,
@ -19,8 +33,6 @@ import {
setupTrayItem,
} from "./utils/main";
let mainWindow: BrowserWindow;
let appIsQuitting = false;
let updateIsAvailable = false;
@ -63,15 +75,68 @@ const setupRendererServer = () => {
serveNextAt(rendererURL);
};
setupRendererServer();
setupLogging(isDev);
function enableSharedArrayBufferSupport() {
app.commandLine.appendSwitch("enable-features", "SharedArrayBuffer");
}
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
app.quit();
} else {
/**
* [Note: Increased disk cache for the desktop app]
*
* Set the "disk-cache-size" command line flag to ask the Chromium process to
* use a larger size for the caches that it keeps on disk. This allows us to use
* the same web-native caching mechanism on both the web and the desktop app,
* just ask the embedded Chromium to be a bit more generous in disk usage when
* running as the desktop app.
*
* The size we provide is in bytes. We set it to a large value, 5 GB (5 * 1024 *
* 1024 * 1024 = 5368709120)
* https://www.electronjs.org/docs/latest/api/command-line-switches#--disk-cache-sizesize
*
* Note that increasing the disk cache size does not guarantee that Chromium
* will respect in verbatim, it uses its own heuristics atop this hint.
* https://superuser.com/questions/378991/what-is-chrome-default-cache-size-limit/1577693#1577693
*/
const increaseDiskCache = () => {
app.commandLine.appendSwitch("disk-cache-size", "5368709120");
};
/**
* Older versions of our app used to maintain a cache dir using the main
* process. This has been deprecated in favor of using a normal web cache (See:
* [Note: Increased disk cache for the desktop app]).
*
* Delete the old cache dir if it exists. This code was added March 2024, and
* can be removed after some time once most people have upgraded to newer
* versions.
*/
const deleteLegacyDiskCacheDirIfExists = async () => {
// The existing code was passing "cache" as a parameter to getPath. This was
// incorrect, cache is not a valid value. However, we replicate that
// behaviour so that we get back the same path that the old got was getting.
// @ts-expect-error
const cacheDir = path.join(app.getPath("cache"), "ente");
if (existsSync(cacheDir)) {
log.info(`Removing legacy disk cache from ${cacheDir}`);
await fs.rm(cacheDir, { recursive: true });
}
};
const main = () => {
setupLogging(isDev);
const gotTheLock = app.requestSingleInstanceLock();
if (!gotTheLock) {
app.quit();
return;
}
let mainWindow: BrowserWindow;
setupRendererServer();
handleDockIconHideOnAutoLaunch();
increaseDiskCache();
enableSharedArrayBufferSupport();
app.on("second-instance", () => {
// Someone tried to run a second instance, we should focus our window.
if (mainWindow) {
@ -99,7 +164,17 @@ if (!gotTheLock) {
handleExternalLinks(mainWindow);
addAllowOriginHeader(mainWindow);
setupAppEventEmitter(mainWindow);
try {
deleteLegacyDiskCacheDirIfExists();
} catch (e) {
// Log but otherwise ignore errors during non-critical startup
// actions
logErrorSentry(e, "Ignoring startup error");
}
});
app.on("before-quit", () => setIsAppQuitting(true));
}
};
main();

3
desktop/src/preload.ts
View file

@ -32,7 +32,6 @@ import { createWriteStream, existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import { Readable } from "node:stream";
import path from "path";
import { deleteDiskCache, openDiskCache } from "./api/cache";
import { logToDisk, openLogDirectory } from "./api/common";
import { runFFmpegCmd } from "./api/ffmpeg";
import { getDirFiles } from "./api/fs";
@ -446,8 +445,6 @@ contextBridge.exposeInMainWorld("ElectronAPIs", {
setToUploadCollection,
getEncryptionKey,
setEncryptionKey,
openDiskCache,
deleteDiskCache,
getDirFiles,
getWatchMappings,
addWatchMapping,

14
desktop/src/services/clipService.ts
View file

@ -63,17 +63,9 @@ const TEXT_MODEL_SIZE_IN_BYTES = {
onnx: 64173509, // 61.2 MB
};
const MODEL_SAVE_FOLDER = "models";
function getModelSavePath(modelName: string) {
let userDataDir: string;
if (isDev) {
userDataDir = ".";
} else {
userDataDir = app.getPath("userData");
}
return path.join(userDataDir, MODEL_SAVE_FOLDER, modelName);
}
/** Return the path where the given {@link modelName} is meant to be saved */
const getModelSavePath = (modelName: string) =>
path.join(app.getPath("userData"), "models", modelName);
async function downloadModel(saveLocation: string, url: string) {
// confirm that the save location exists

59
desktop/src/services/diskCache.ts
View file

@ -1,59 +0,0 @@
import { existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import DiskLRUService from "../services/diskLRU";
import { LimitedCache } from "../types/cache";
import { getFileStream, writeStream } from "./fs";
import { logError } from "./logging";
const DEFAULT_CACHE_LIMIT = 1000 * 1000 * 1000; // 1GB
export class DiskCache implements LimitedCache {
constructor(
private cacheBucketDir: string,
private cacheLimit = DEFAULT_CACHE_LIMIT,
) {}
async put(cacheKey: string, response: Response): Promise<void> {
const cachePath = path.join(this.cacheBucketDir, cacheKey);
await writeStream(cachePath, response.body);
DiskLRUService.enforceCacheSizeLimit(
this.cacheBucketDir,
this.cacheLimit,
);
}
async match(
cacheKey: string,
{ sizeInBytes }: { sizeInBytes?: number } = {},
): Promise<Response> {
const cachePath = path.join(this.cacheBucketDir, cacheKey);
if (existsSync(cachePath)) {
const fileStats = await fs.stat(cachePath);
if (sizeInBytes && fileStats.size !== sizeInBytes) {
logError(
Error(),
"Cache key exists but size does not match. Deleting cache key.",
);
fs.unlink(cachePath).catch((e) => {
if (e.code === "ENOENT") return;
logError(e, "Failed to delete cache key");
});
return undefined;
}
DiskLRUService.markUse(cachePath);
return new Response(await getFileStream(cachePath));
} else {
return undefined;
}
}
async delete(cacheKey: string): Promise<boolean> {
const cachePath = path.join(this.cacheBucketDir, cacheKey);
if (existsSync(cachePath)) {
await fs.unlink(cachePath);
return true;
} else {
return false;
}
}
}

95
desktop/src/services/diskLRU.ts
View file

@ -1,95 +0,0 @@
import getFolderSize from "get-folder-size";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { logError } from "../services/logging";
export interface LeastRecentlyUsedResult {
atime: Date;
path: string;
}
class DiskLRUService {
private isRunning: Promise<any> = null;
private reRun: boolean = false;
/** Mark "use" of a given file by updating its modified time */
async markUse(path: string) {
const now = new Date();
await fs.utimes(path, now, now);
}
enforceCacheSizeLimit(cacheDir: string, maxSize: number) {
if (!this.isRunning) {
this.isRunning = this.evictLeastRecentlyUsed(cacheDir, maxSize);
this.isRunning.then(() => {
this.isRunning = null;
if (this.reRun) {
this.reRun = false;
this.enforceCacheSizeLimit(cacheDir, maxSize);
}
});
} else {
this.reRun = true;
}
}
async evictLeastRecentlyUsed(cacheDir: string, maxSize: number) {
try {
await new Promise((resolve) => {
getFolderSize(cacheDir, async (err, size) => {
if (err) {
throw err;
}
if (size >= maxSize) {
const leastRecentlyUsed =
await this.findLeastRecentlyUsed(cacheDir);
try {
await fs.unlink(leastRecentlyUsed.path);
} catch (e) {
// ENOENT: File not found
// which can be ignored as we are trying to delete the file anyway
if (e.code !== "ENOENT") {
logError(
e,
"Failed to evict least recently used",
);
}
// ignoring the error, as it would get retried on the next run
}
this.evictLeastRecentlyUsed(cacheDir, maxSize);
}
resolve(null);
});
});
} catch (e) {
logError(e, "evictLeastRecentlyUsed failed");
}
}
private async findLeastRecentlyUsed(
dir: string,
result?: LeastRecentlyUsedResult,
): Promise<LeastRecentlyUsedResult> {
result = result || { atime: new Date(), path: "" };
const files = await fs.readdir(dir);
for (const file of files) {
const newBase = path.join(dir, file);
const st = await fs.stat(newBase);
if (st.isDirectory()) {
result = await this.findLeastRecentlyUsed(newBase, result);
} else {
const { atime } = st;
if (st.atime.getTime() < result.atime.getTime()) {
result = {
atime,
path: newBase,
};
}
}
}
return result;
}
}
export default new DiskLRUService();

8
desktop/src/types/cache.ts
View file

@ -1,8 +0,0 @@
export interface LimitedCache {
match: (
key: string,
options?: { sizeInBytes?: number },
) => Promise<Response>;
put: (key: string, data: Response) => Promise<void>;
delete: (key: string) => Promise<boolean>;
}

16
desktop/src/utils/ipcComms.ts
View file

@ -95,7 +95,13 @@ export default function setupIpcComs(
clearElectronStore();
});
ipcMain.handle("get-path", (_, message) => {
ipcMain.handle("convert-to-jpeg", (_, fileData, filename) => {
return convertToJPEG(fileData, filename);
});
ipcMain.handle("open-log-dir", () => {
// [Note: Electron app paths]
//
// By default, these paths are at the following locations:
//
// * macOS: `~/Library/Application Support/ente`
@ -104,14 +110,6 @@ export default function setupIpcComs(
// * Windows: C:\Users\<you>\AppData\Local\<Your App Name>
//
// https://www.electronjs.org/docs/latest/api/app
return app.getPath(message);
});
ipcMain.handle("convert-to-jpeg", (_, fileData, filename) => {
return convertToJPEG(fileData, filename);
});
ipcMain.handle("open-log-dir", () => {
shell.openPath(app.getPath("logs"));
});

4
desktop/src/utils/main.ts
View file

@ -94,10 +94,6 @@ export async function handleDockIconHideOnAutoLaunch() {
}
}
export function enableSharedArrayBufferSupport() {
app.commandLine.appendSwitch("enable-features", "SharedArrayBuffer");
}
export function logSystemInfo() {
const systemVersion = process.getSystemVersion();
const osName = process.platform;

2
desktop/src/utils/temp.ts
View file

@ -1,4 +1,4 @@
import { app } from "electron";
import { app } from "electron/main";
import { existsSync } from "node:fs";
import * as fs from "node:fs/promises";
import path from "path";

23
desktop/yarn.lock
View file

@ -261,11 +261,6 @@
dependencies:
"@types/node" "*"
"@types/get-folder-size@^2.0.0":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@types/get-folder-size/-/get-folder-size-2.0.0.tgz#acbb5bf5999410c375b2739863a9d2f9483fabf6"
integrity sha512-6VKKrDB20E/6ovi2Pfpy9Pcz8Me1ue/tReaZrwrz9mfVdsr6WAMiDZ+F1oAAcss4U5n2k673i1leDIx2aEBDFQ==
"@types/http-cache-semantics@*":
version "4.0.2"
resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz#abe102d06ccda1efdf0ed98c10ccf7f36a785a41"
@ -1535,24 +1530,11 @@ function-bind@^1.1.1:
resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==
gar@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/gar/-/gar-1.0.4.tgz#f777bc7db425c0572fdeb52676172ca1ae9888b8"
integrity sha512-w4n9cPWyP7aHxKxYHFQMegj7WIAsL/YX/C4Bs5Rr8s1H9M1rNtRWRsw+ovYMkXDQ5S4ZbYHsHAPmevPjPgw44w==
get-caller-file@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==
get-folder-size@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/get-folder-size/-/get-folder-size-2.0.1.tgz#3fe0524dd3bad05257ef1311331417bcd020a497"
integrity sha512-+CEb+GDCM7tkOS2wdMKTn9vU7DgnKUTuDlehkNJKNSovdCOVxs14OfKCk4cvSaR3za4gj+OBdl9opPN9xrJ0zA==
dependencies:
gar "^1.0.4"
tiny-each-async "2.0.3"
get-intrinsic@^1.1.1:
version "1.1.2"
resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.2.tgz#336975123e05ad0b7ba41f152ee4aadbea6cf598"
@ -2888,11 +2870,6 @@ text-table@^0.2.0:
resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==
tiny-each-async@2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/tiny-each-async/-/tiny-each-async-2.0.3.tgz#8ebbbfd6d6295f1370003fbb37162afe5a0a51d1"
integrity sha512-5ROII7nElnAirvFn8g7H7MtpfV1daMcyfTGQwsn/x2VtyV+VPiO5CjReCJtWLvoKTDEDmZocf3cNPraiMnBXLA==
tmp-promise@^3.0.2:
version "3.0.3"
resolved "https://registry.yarnpkg.com/tmp-promise/-/tmp-promise-3.0.3.tgz#60a1a1cc98c988674fcbfd23b6e3367bdeac4ce7"