beenull/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Inline

Browse source 
The zip-file tests are no longer directly usable, the way we read the files has
changed, these will have to be recreated in a new form.
This commit is contained in:
Manav Rathi 2024-04-27 17:42:36 +05:30
parent 6671a62c78
commit bb2ddec163
No known key found for this signature in database
5 changed files with 24 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
web/apps/photos/src/components/Sidebar/DebugSection.tsx
View file

@ -9,10 +9,6 @@ import { useContext, useEffect, useState } from "react";
import { Trans } from "react-i18next";
import { isInternalUser } from "utils/user";
import { testUpload } from "../../../tests/upload.test";
import {
testZipFileReading,
testZipWithRootFileReadingTest,
} from "../../../tests/zip-file-reading.test";
export default function DebugSection() {
const appContext = useContext(AppContext);
@ -62,25 +58,11 @@ export default function DebugSection() {
</Typography>
)}
{isInternalUser() && (
<>
<EnteMenuItem
variant="secondary"
onClick={testUpload}
label={"Test Upload"}
/>
<EnteMenuItem
variant="secondary"
onClick={testZipFileReading}
label="Test Zip file reading"
/>
<EnteMenuItem
variant="secondary"
onClick={testZipWithRootFileReadingTest}
label="Zip with Root file Test"
/>
</>
<EnteMenuItem
variant="secondary"
onClick={testUpload}
label={"Test Upload"}
/>
)}
</>
);

19
web/apps/photos/src/services/upload/uploadService.ts
View file

@ -71,6 +71,25 @@ const maximumChunksPerUploadPart = 5;
* */
const multipartPartSize = ENCRYPTION_CHUNK_SIZE * maximumChunksPerUploadPart;
interface DataStream {
stream: ReadableStream<Uint8Array>;
chunkCount: number;
}
function isDataStream(object: any): object is DataStream {
return "stream" in object;
}
interface LocalFileAttributes<T extends string | Uint8Array | DataStream> {
encryptedData: T;
decryptionHeader: string;
}
interface EncryptionResult<T extends string | Uint8Array | DataStream> {
file: LocalFileAttributes<T>;
key: string;
}
/** Upload files to cloud storage */
class UploadService {
private uploadURLs: UploadURL[] = [];

112
web/apps/photos/tests/zip-file-reading.test.ts
View file

@ -1,112 +0,0 @@
import { getFileNameSize } from "@/next/file";
import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/constants";
import type { DataStream } from "@ente/shared/utils/data-stream";
import { PICKED_UPLOAD_TYPE } from "constants/upload";
import { getElectronFileStream, getFileStream } from "services/readerService";
import { getImportSuggestion } from "utils/upload";
// This was for used to verify that converting from the browser readable stream
// to the node readable stream correctly handles files that align on the 4 MB
// data boundary. This expects a zip file containing random files of various
// sizes starting from 1M to 20M.
export const testZipFileReading = async () => {
try {
const electron = globalThis.electron;
if (!electron) {
console.log("testZipFileReading Check is for desktop only");
return;
}
if (!process.env.NEXT_PUBLIC_FILE_READING_TEST_ZIP_PATH) {
throw Error(
"upload test failed NEXT_PUBLIC_FILE_READING_TEST_ZIP_PATH missing",
);
}
const files = await electron.getElectronFilesFromGoogleZip(
process.env.NEXT_PUBLIC_FILE_READING_TEST_ZIP_PATH,
);
if (!files?.length) {
throw Error(
`testZipFileReading Check failed ❌
No files selected`,
);
}
console.log("test zip file reading check started");
let i = 0;
for (const file of files) {
i++;
let filedata: DataStream;
if (file instanceof File) {
filedata = getFileStream(file, ENCRYPTION_CHUNK_SIZE);
} else {
filedata = await getElectronFileStream(
file,
ENCRYPTION_CHUNK_SIZE,
);
}
const streamReader = filedata.stream.getReader();
for (let i = 0; i < filedata.chunkCount; i++) {
const { done } = await streamReader.read();
if (done) {
throw Error(
`testZipFileReading Check failed ❌
${getFileNameSize(
file,
)} less than expected chunks, expected: ${
filedata.chunkCount
}, got ${i - 1}`,
);
}
}
const { done } = await streamReader.read();
if (!done) {
throw Error(
`testZipFileReading Check failed ❌
${getFileNameSize(
file,
)} more than expected chunks, expected: ${
filedata.chunkCount
}`,
);
}
console.log(`${i}/${files.length} passed ✅`);
}
console.log("test zip file reading check passed ✅");
} catch (e) {
console.log(e);
}
};
// This was used when fixing a bug around handling a zip file that has a photo
// at the root.
export const testZipWithRootFileReadingTest = async () => {
try {
const electron = globalThis.electron;
if (!electron) {
console.log("testZipFileReading Check is for desktop only");
return;
}
if (!process.env.NEXT_PUBLIC_ZIP_WITH_ROOT_FILE_PATH) {
throw Error(
"upload test failed NEXT_PUBLIC_ZIP_WITH_ROOT_FILE_PATH missing",
);
}
const files = await electron.getElectronFilesFromGoogleZip(
process.env.NEXT_PUBLIC_ZIP_WITH_ROOT_FILE_PATH,
);
const importSuggestion = getImportSuggestion(
PICKED_UPLOAD_TYPE.ZIPS,
files.map((file) => file["path"]),
);
if (!importSuggestion.rootFolderName) {
throw Error(
`testZipWithRootFileReadingTest Check failed ❌
rootFolderName is missing`,
);
}
console.log("testZipWithRootFileReadingTest passed ✅");
} catch (e) {
console.log(e);
}
};

14
web/packages/shared/crypto/types.ts
View file

@ -1,17 +1,3 @@
import type { DataStream } from "../utils/data-stream";
export interface LocalFileAttributes<
T extends string | Uint8Array | DataStream,
> {
encryptedData: T;
decryptionHeader: string;
}
export interface EncryptionResult<T extends string | Uint8Array | DataStream> {
file: LocalFileAttributes<T>;
key: string;
}
export interface B64EncryptionResult {
encryptedData: string;
key: string;

8
web/packages/shared/utils/data-stream.ts
View file

@ -1,8 +0,0 @@
export interface DataStream {
stream: ReadableStream<Uint8Array>;
chunkCount: number;
}
export function isDataStream(object: any): object is DataStream {
return "stream" in object;
}