0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Dedup comlink

Browse source
This commit is contained in:
Manav Rathi 2024-04-08 20:50:14 +05:30
parent a1c9ceae6b
commit 92a3650696
No known key found for this signature in database
14 changed files with 11 additions and 276 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
web/apps/cast/src/services/castDownloadManager.ts
View file

@ -1,9 +1,9 @@
import ComlinkCryptoWorker from "@ente/shared/crypto";
import { CustomError } from "@ente/shared/error";
import HTTPService from "@ente/shared/network/HTTPService";
import { getCastFileURL } from "@ente/shared/network/api";
import { FILE_TYPE } from "constants/file";
import { EnteFile } from "types/file";
import ComlinkCryptoWorker from "utils/comlink/ComlinkCryptoWorker";
import { generateStreamFromArrayBuffer } from "utils/file";
class CastDownloadManager {

25
web/apps/cast/src/utils/comlink/ComlinkCryptoWorker.ts
View file

@ -1,25 +0,0 @@
import { Remote } from "comlink";
import { DedicatedCryptoWorker } from "worker/crypto.worker";
import { ComlinkWorker } from "./comlinkWorker";
class ComlinkCryptoWorker {
private comlinkWorkerInstance: Promise<Remote<DedicatedCryptoWorker>>;
async getInstance() {
if (!this.comlinkWorkerInstance) {
const comlinkWorker = getDedicatedCryptoWorker();
this.comlinkWorkerInstance = comlinkWorker.remote;
}
return this.comlinkWorkerInstance;
}
}
export const getDedicatedCryptoWorker = () => {
const cryptoComlinkWorker = new ComlinkWorker<typeof DedicatedCryptoWorker>(
"ente-crypto-worker",
new Worker(new URL("worker/crypto.worker.ts", import.meta.url)),
);
return cryptoComlinkWorker;
};
export default new ComlinkCryptoWorker();

25
web/apps/cast/src/utils/comlink/comlinkWorker.ts
View file

@ -1,25 +0,0 @@
import log from "@/next/log";
import { Remote, wrap } from "comlink";
export class ComlinkWorker<T extends new () => InstanceType<T>> {
public remote: Promise<Remote<InstanceType<T>>>;
private worker: Worker;
private name: string;
constructor(name: string, worker: Worker) {
this.name = name;
this.worker = worker;
this.worker.onerror = (errorEvent) => {
console.error("Got error event from worker", errorEvent);
};
log.debug(() => `Initiated ${this.name}`);
const comlink = wrap<T>(this.worker);
this.remote = new comlink() as Promise<Remote<InstanceType<T>>>;
}
public terminate() {
this.worker.terminate();
log.debug(() => `Terminated ${this.name}`);
}
}

2
web/apps/cast/src/utils/file/index.ts
View file

@ -1,3 +1,4 @@
import ComlinkCryptoWorker from "@ente/shared/crypto";
import { logError } from "@ente/shared/sentry";
import { FILE_TYPE, RAW_FORMATS } from "constants/file";
import CastDownloadManager from "services/castDownloadManager";
@ -9,7 +10,6 @@ import {
FileMagicMetadata,
FilePublicMagicMetadata,
} from "types/file";
import ComlinkCryptoWorker from "utils/comlink/ComlinkCryptoWorker";
export function sortFiles(files: EnteFile[], sortAsc = false) {
// sort based on the time of creation time of the file,

215
web/apps/cast/src/worker/crypto.worker.ts
View file

@ -1,215 +0,0 @@
import * as libsodium from "@ente/shared/crypto/internal/libsodium";
import * as Comlink from "comlink";
import { StateAddress } from "libsodium-wrappers";
const textDecoder = new TextDecoder();
const textEncoder = new TextEncoder();
export class DedicatedCryptoWorker {
async decryptMetadata(
encryptedMetadata: string,
header: string,
key: string,
) {
const encodedMetadata = await libsodium.decryptChaChaOneShot(
await libsodium.fromB64(encryptedMetadata),
await libsodium.fromB64(header),
key,
);
return JSON.parse(textDecoder.decode(encodedMetadata));
}
async decryptThumbnail(
fileData: Uint8Array,
header: Uint8Array,
key: string,
) {
return libsodium.decryptChaChaOneShot(fileData, header, key);
}
async decryptEmbedding(
encryptedEmbedding: string,
header: string,
key: string,
) {
const encodedEmbedding = await libsodium.decryptChaChaOneShot(
await libsodium.fromB64(encryptedEmbedding),
await libsodium.fromB64(header),
key,
);
return Float32Array.from(
JSON.parse(textDecoder.decode(encodedEmbedding)),
);
}
async decryptFile(fileData: Uint8Array, header: Uint8Array, key: string) {
return libsodium.decryptChaCha(fileData, header, key);
}
async encryptMetadata(metadata: Object, key: string) {
const encodedMetadata = textEncoder.encode(JSON.stringify(metadata));
const { file: encryptedMetadata } =
await libsodium.encryptChaChaOneShot(encodedMetadata, key);
const { encryptedData, ...other } = encryptedMetadata;
return {
file: {
encryptedData: await libsodium.toB64(encryptedData),
...other,
},
key,
};
}
async encryptThumbnail(fileData: Uint8Array, key: string) {
return libsodium.encryptChaChaOneShot(fileData, key);
}
async encryptEmbedding(embedding: Float32Array, key: string) {
const encodedEmbedding = textEncoder.encode(
JSON.stringify(Array.from(embedding)),
);
const { file: encryptEmbedding } = await libsodium.encryptChaChaOneShot(
encodedEmbedding,
key,
);
const { encryptedData, ...other } = encryptEmbedding;
return {
file: {
encryptedData: await libsodium.toB64(encryptedData),
...other,
},
key,
};
}
async encryptFile(fileData: Uint8Array) {
return libsodium.encryptChaCha(fileData);
}
async encryptFileChunk(
data: Uint8Array,
pushState: StateAddress,
isFinalChunk: boolean,
) {
return libsodium.encryptFileChunk(data, pushState, isFinalChunk);
}
async initChunkEncryption() {
return libsodium.initChunkEncryption();
}
async initChunkDecryption(header: Uint8Array, key: Uint8Array) {
return libsodium.initChunkDecryption(header, key);
}
async decryptFileChunk(fileData: Uint8Array, pullState: StateAddress) {
return libsodium.decryptFileChunk(fileData, pullState);
}
async initChunkHashing() {
return libsodium.initChunkHashing();
}
async hashFileChunk(hashState: StateAddress, chunk: Uint8Array) {
return libsodium.hashFileChunk(hashState, chunk);
}
async completeChunkHashing(hashState: StateAddress) {
return libsodium.completeChunkHashing(hashState);
}
async deriveKey(
passphrase: string,
salt: string,
opsLimit: number,
memLimit: number,
) {
return libsodium.deriveKey(passphrase, salt, opsLimit, memLimit);
}
async deriveSensitiveKey(passphrase: string, salt: string) {
return libsodium.deriveSensitiveKey(passphrase, salt);
}
async deriveInteractiveKey(passphrase: string, salt: string) {
return libsodium.deriveInteractiveKey(passphrase, salt);
}
async decryptB64(data: string, nonce: string, key: string) {
return libsodium.decryptB64(data, nonce, key);
}
async decryptToUTF8(data: string, nonce: string, key: string) {
return libsodium.decryptToUTF8(data, nonce, key);
}
async encryptToB64(data: string, key: string) {
return libsodium.encryptToB64(data, key);
}
async generateKeyAndEncryptToB64(data: string) {
return libsodium.generateKeyAndEncryptToB64(data);
}
async encryptUTF8(data: string, key: string) {
return libsodium.encryptUTF8(data, key);
}
async generateEncryptionKey() {
return libsodium.generateEncryptionKey();
}
async generateSaltToDeriveKey() {
return libsodium.generateSaltToDeriveKey();
}
async generateKeyPair() {
return libsodium.generateKeyPair();
}
async boxSealOpen(input: string, publicKey: string, secretKey: string) {
return libsodium.boxSealOpen(input, publicKey, secretKey);
}
async boxSeal(input: string, publicKey: string) {
return libsodium.boxSeal(input, publicKey);
}
async generateSubKey(
key: string,
subKeyLength: number,
subKeyID: number,
context: string,
) {
return libsodium.generateSubKey(key, subKeyLength, subKeyID, context);
}
async fromUTF8(string: string) {
return libsodium.fromUTF8(string);
}
async toUTF8(data: string) {
return libsodium.toUTF8(data);
}
async toB64(data: Uint8Array) {
return libsodium.toB64(data);
}
async toURLSafeB64(data: Uint8Array) {
return libsodium.toURLSafeB64(data);
}
async fromB64(string: string) {
return libsodium.fromB64(string);
}
async toHex(string: string) {
return libsodium.toHex(string);
}
async fromHex(string: string) {
return libsodium.fromHex(string);
}
}
Comlink.expose(DedicatedCryptoWorker, self);

2
web/apps/photos/src/services/heic-convert/service.ts
View file

@ -1,10 +1,10 @@
import { convertBytesToHumanReadable } from "@/next/file";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { CustomError } from "@ente/shared/error";
import { addLogLine } from "@ente/shared/logging";
import { logError } from "@ente/shared/sentry";
import { retryAsyncFunction } from "@ente/shared/utils";
import QueueProcessor from "@ente/shared/utils/queueProcessor";
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import { getDedicatedConvertWorker } from "utils/comlink/ComlinkConvertWorker";
import { DedicatedConvertWorker } from "worker/convert.worker";

2
web/apps/photos/src/services/machineLearning/machineLearningFactory.ts
View file

@ -1,8 +1,8 @@
import { haveWindow } from "@/next/env";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { getDedicatedCryptoWorker } from "@ente/shared/crypto";
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
import { addLogLine } from "@ente/shared/logging";
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import PQueue from "p-queue";
import { EnteFile } from "types/file";
import {

2
web/apps/photos/src/services/machineLearning/mlWorkManager.ts
View file

@ -2,7 +2,7 @@ import { eventBus, Events } from "@ente/shared/events";
import { addLogLine } from "@ente/shared/logging";
import { logError } from "@ente/shared/sentry";
import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers";
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { FILE_TYPE } from "constants/file";
import debounce from "debounce";
import PQueue from "p-queue";

2
web/apps/photos/src/services/upload/uploadManager.ts
View file

@ -28,7 +28,7 @@ import { getFileNameSize } from "@/next/file";
import { getDedicatedCryptoWorker } from "@ente/shared/crypto";
import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker";
import { addLogLine } from "@ente/shared/logging";
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { Remote } from "comlink";
import { UPLOAD_RESULT, UPLOAD_STAGES } from "constants/upload";
import isElectron from "is-electron";

2
web/apps/photos/src/utils/comlink/ComlinkConvertWorker.ts
View file

@ -1,5 +1,5 @@
import { haveWindow } from "@/next/env";
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { Remote } from "comlink";
import { DedicatedConvertWorker } from "worker/convert.worker";

2
web/apps/photos/src/utils/comlink/ComlinkFFmpegWorker.ts
View file

@ -1,4 +1,4 @@
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { Remote } from "comlink";
import { DedicatedFFmpegWorker } from "worker/ffmpeg.worker";

2
web/apps/photos/src/utils/comlink/ComlinkMLWorker.ts
View file

@ -1,5 +1,5 @@
import { haveWindow } from "@/next/env";
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { DedicatedMLWorker } from "worker/ml.worker";
export const getDedicatedMLWorker = (name: string) => {

2
web/apps/photos/src/utils/comlink/ComlinkSearchWorker.ts
View file

@ -1,5 +1,5 @@
import { haveWindow } from "@/next/env";
import { ComlinkWorker } from "@ente/shared/worker/comlinkWorker";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { Remote } from "comlink";
import { DedicatedSearchWorker } from "worker/search.worker";

2
web/packages/shared/crypto/index.ts
View file

@ -1,4 +1,4 @@
import { ComlinkWorker } from "@/next/worker/comlinkWorker";
import { ComlinkWorker } from "@/next/worker/comlink-worker";
import { Remote } from "comlink";
import { DedicatedCryptoWorker } from "./internal/crypto.worker";