0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #27 from ente-io/fixing-upload-memory-leak

Browse source 
Fixing upload memory leak
This commit is contained in:
Vishnu Mohandas 2021-02-26 00:21:51 +05:30 • committed by GitHub
commit 11ace7d793
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 58 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/pages/credentials/index.tsx
View file

@ -11,11 +11,7 @@ import { useRouter } from 'next/router';
import * as Yup from 'yup';
import { keyAttributes } from 'types';
import { setKey, SESSION_KEYS, getKey } from 'utils/storage/sessionStorage';
import * as Comlink from 'comlink';
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
import CryptoWorker from 'utils/crypto/cryptoWorker';
const Image = styled.img`
width: 200px;

1
src/pages/gallery/components/AddCollection.tsx
View file

@ -42,7 +42,6 @@ export default function AddCollection(props) {
<div style={{ margin: '10px' }}>
<DropzoneWrapper
onDropAccepted={createCollection}
onDropRejected={closeUploadModal}
onDragOver={showUploadModal}
children={children}
/>

8
src/pages/gallery/components/CollectionDropZone.tsx
View file

@ -33,13 +33,7 @@ function CollectionDropZone({
setProgressView(false);
}
};
return (
<DropzoneWrapper
children={children}
onDropAccepted={upload}
onDropRejected={closeModal}
/>
);
return <DropzoneWrapper children={children} onDropAccepted={upload} />;
}
export default CollectionDropZone;

6
src/pages/generate/index.tsx
View file

@ -11,12 +11,8 @@ import { putAttributes } from 'services/userService';
import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
import { useRouter } from 'next/router';
import { getKey, SESSION_KEYS, setKey } from 'utils/storage/sessionStorage';
import * as Comlink from 'comlink';
import { B64EncryptionResult } from 'services/uploadService';
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
import CryptoWorker from 'utils/crypto/cryptoWorker';
const Image = styled.img`
width: 200px;

5
src/services/collectionService.ts
View file

@ -4,14 +4,11 @@ import { file } from './fileService';
import localForage from 'localforage';
import HTTPService from './HTTPService';
import * as Comlink from 'comlink';
import { B64EncryptionResult } from './uploadService';
import { getActualKey, getToken } from 'utils/common/key';
import { user } from './userService';
import CryptoWorker from 'utils/crypto/cryptoWorker';
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
const ENDPOINT = getEndpoint();
enum CollectionType {

6
src/services/downloadManager.ts
View file

@ -2,13 +2,11 @@ import { getToken } from 'utils/common/key';
import { file } from './fileService';
import HTTPService from './HTTPService';
import { getEndpoint } from 'utils/common/apiUtil';
import * as Comlink from 'comlink';
import { getFileExtension } from 'utils/common/utilFunctions';
import CryptoWorker from 'utils/crypto/cryptoWorker';
const ENDPOINT = getEndpoint();
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
const heic2any = typeof window !== 'undefined' && require('heic2any');
const TYPE_HEIC = 'heic';

5
src/services/fileService.ts
View file

@ -1,13 +1,10 @@
import { getEndpoint } from 'utils/common/apiUtil';
import HTTPService from './HTTPService';
import * as Comlink from 'comlink';
import localForage from 'localforage';
import { collection } from './collectionService';
import { MetadataObject } from './uploadService';
import CryptoWorker from 'utils/crypto/cryptoWorker';
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
const ENDPOINT = getEndpoint();
const DIFF_LIMIT: number = 2500;

63
src/services/uploadService.ts
View file

@ -1,15 +1,12 @@
import { getEndpoint } from 'utils/common/apiUtil';
import HTTPService from './HTTPService';
import * as Comlink from 'comlink';
import EXIF from 'exif-js';
import { fileAttribute } from './fileService';
import { collection, CollectionAndItsLatestFile } from './collectionService';
import { FILE_TYPE } from 'pages/gallery';
import { checkConnectivity } from 'utils/common/utilFunctions';
import { ErrorHandler } from 'utils/common/errorUtil';
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
import CryptoWorker from 'utils/crypto/cryptoWorker';
const ENDPOINT = getEndpoint();
const THUMBNAIL_HEIGHT = 720;
@ -135,6 +132,7 @@ class UploadService {
uploadProcesses.push(
this.uploader(
await new CryptoWorker(),
new FileReader(),
this.filesToBeUploaded.pop(),
collectionAndItsLatestFile.collection,
token
@ -150,25 +148,38 @@ class UploadService {
throw e;
}
}
private async uploader(worker, rawFile, collection, token) {
private async uploader(
worker: any,
reader: FileReader,
rawFile: File,
collection: collection,
token: string
) {
try {
let file: FileinMemory = await this.readFile(rawFile);
let encryptedFile: EncryptedFile = await this.encryptFile(
let file: FileinMemory = await this.readFile(reader, rawFile);
let {
file: encryptedFile,
fileKey: encryptedKey,
}: EncryptedFile = await this.encryptFile(
worker,
file,
collection.key
);
file = null;
let backupedFile: BackupedFile = await this.uploadtoBucket(
encryptedFile.file,
encryptedFile,
token
);
encryptedFile = null;
let uploadFile: uploadFile = this.getuploadFile(
collection,
backupedFile,
encryptedFile.fileKey
encryptedKey
);
encryptedKey = null;
backupedFile = null;
await this.uploadFile(uploadFile, token);
uploadFile = null;
this.filesCompleted++;
this.changeProgressBarProps();
} catch (e) {
@ -177,11 +188,11 @@ class UploadService {
`Uploading Failed for File - ${rawFile.name}`
);
this.uploadErrors.push(error);
this.setUploadErrors(this.uploadErrors);
}
if (this.filesToBeUploaded.length > 0) {
await this.uploader(
worker,
reader,
this.filesToBeUploaded.pop(),
collection,
token
@ -196,14 +207,19 @@ class UploadService {
total: this.totalFileCount,
});
setPercentComplete(this.filesCompleted * this.perFileProgress);
this.setUploadErrors(this.uploadErrors);
}
private async readFile(recievedFile: File) {
private async readFile(reader: FileReader, recievedFile: File) {
try {
const filedata: Uint8Array = await this.getUint8ArrayView(
reader,
recievedFile
);
const thumbnail = await this.generateThumbnail(
reader,
recievedFile
);
const thumbnail = await this.generateThumbnail(recievedFile);
let fileType: FILE_TYPE;
switch (recievedFile.type.split('/')[0]) {
@ -218,6 +234,7 @@ class UploadService {
}
const { location, creationTime } = await this.getExifData(
reader,
recievedFile
);
const metadata = Object.assign(
@ -389,7 +406,10 @@ class UploadService {
this.setUploadErrors(this.uploadErrors);
}
}
private async generateThumbnail(file: File): Promise<Uint8Array> {
private async generateThumbnail(
reader: FileReader,
file: File
): Promise<Uint8Array> {
try {
let canvas = document.createElement('canvas');
let canvas_CTX = canvas.getContext('2d');
@ -469,7 +489,10 @@ class UploadService {
thumbnailBlob.size > MIN_THUMBNAIL_SIZE &&
attempts <= MAX_ATTEMPTS
);
const thumbnail = await this.getUint8ArrayView(thumbnailBlob);
const thumbnail = await this.getUint8ArrayView(
reader,
thumbnailBlob
);
return thumbnail;
} catch (e) {
console.log('Error generating thumbnail ', e);
@ -477,11 +500,12 @@ class UploadService {
}
}
private async getUint8ArrayView(file): Promise<Uint8Array> {
private async getUint8ArrayView(
reader: FileReader,
file: Blob
): Promise<Uint8Array> {
try {
return await new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onabort = () => reject('file reading was aborted');
reader.onerror = () => reject('file reading has failed');
reader.onload = () => {
@ -548,10 +572,9 @@ class UploadService {
}
}
private async getExifData(recievedFile) {
private async getExifData(reader: FileReader, recievedFile: File) {
try {
const exifData: any = await new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
resolve(EXIF.readFromBinaryFile(reader.result));
};

6
src/utils/common/key.ts
View file

@ -1,10 +1,6 @@
import CryptoWorker from 'utils/crypto/cryptoWorker';
import { getData, LS_KEYS } from 'utils/storage/localStorage';
import { getKey, SESSION_KEYS } from 'utils/storage/sessionStorage';
import * as Comlink from 'comlink';
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
export const getActualKey = async () => {
const session = getData(LS_KEYS.SESSION);

7
src/utils/crypto/cryptoWorker.ts Normal file
View file

@ -0,0 +1,7 @@
import * as Comlink from 'comlink';
const CryptoWorker: any =
typeof window !== 'undefined' &&
Comlink.wrap(new Worker('worker/crypto.worker.js', { type: 'module' }));
export default CryptoWorker;