From 39737b985b2df396f096f6c0b2b93f702d865f7c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 29 Apr 2024 21:38:21 +0530 Subject: [PATCH] teach readstream about zips --- .../photos/src/services/upload/takeout.ts | 24 ++++++--- .../src/services/upload/uploadManager.ts | 53 ++++++++----------- web/apps/photos/src/utils/native-stream.ts | 24 ++++++--- 3 files changed, 56 insertions(+), 45 deletions(-) diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index 5cd16130e..2a71e420a 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -5,6 +5,8 @@ import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; import { NULL_LOCATION } from "constants/upload"; import type { Location } from "types/metadata"; +import { readStream } from "utils/native-stream"; +import type { UploadItem } from "./uploadManager"; export interface ParsedMetadataJSON { creationTime: number; @@ -75,21 +77,29 @@ function getFileOriginalName(fileName: string) { /** Try to parse the contents of a metadata JSON file from a Google Takeout. */ export const tryParseTakeoutMetadataJSON = async ( - fileOrPath: File | string, + uploadItem: UploadItem, ): Promise => { try { - const text = - fileOrPath instanceof File - ? await fileOrPath.text() - : await ensureElectron().fs.readTextFile(fileOrPath); - - return parseMetadataJSONText(text); + return parseMetadataJSONText(await uploadItemText(uploadItem)); } catch (e) { log.error("Failed to parse takeout metadata JSON", e); return undefined; } }; +const uploadItemText = async (uploadItem: UploadItem) => { + if (uploadItem instanceof File) { + return await uploadItem.text(); + } else if (typeof uploadItem == "string") { + return await ensureElectron().fs.readTextFile(uploadItem); + } else if (Array.isArray(uploadItem)) { + const { response } = await readStream(ensureElectron(), uploadItem); + return await response.text(); + } else { + return await uploadItem.file.text(); + } +}; + const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { creationTime: null, modificationTime: null, diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index d5a7b4caf..b561cb02f 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -3,7 +3,7 @@ import { potentialFileTypeFromExtension } from "@/media/live-photo"; import { ensureElectron } from "@/next/electron"; import { lowercaseExtension, nameAndExtension } from "@/next/file"; import log from "@/next/log"; -import { ElectronFile, type FileAndPath } from "@/next/types/file"; +import { type FileAndPath } from "@/next/types/file"; import type { Electron, ZipEntry } from "@/next/types/ipc"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; @@ -437,25 +437,25 @@ class UploadManager { try { await this.updateExistingFilesAndCollections(collections); - const namedFiles = itemsWithCollection.map( + const namedItems = itemsWithCollection.map( makeUploadItemWithCollectionIDAndName, ); - this.uiService.setFiles(namedFiles); + this.uiService.setFiles(namedItems); - const [metadataFiles, mediaFiles] = - splitMetadataAndMediaFiles(namedFiles); + const [metadataItems, mediaItems] = + splitMetadataAndMediaItems(namedItems); - if (metadataFiles.length) { + if (metadataItems.length) { this.uiService.setUploadStage( UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES, ); - await this.parseMetadataJSONFiles(metadataFiles); + await this.parseMetadataJSONFiles(metadataItems); } - if (mediaFiles.length) { - const clusteredMediaFiles = await clusterLivePhotos(mediaFiles); + if (mediaItems.length) { + const clusteredMediaFiles = await clusterLivePhotos(mediaItems); this.abortIfCancelled(); @@ -464,7 +464,7 @@ class UploadManager { this.uiService.setFiles(clusteredMediaFiles); this.uiService.setHasLivePhoto( - mediaFiles.length != clusteredMediaFiles.length, + mediaItems.length != clusteredMediaFiles.length, ); await this.uploadMediaFiles(clusteredMediaFiles); @@ -510,19 +510,17 @@ class UploadManager { } private async parseMetadataJSONFiles( - files: UploadItemWithCollectionIDAndName[], + items: UploadItemWithCollectionIDAndName[], ) { - this.uiService.reset(files.length); + this.uiService.reset(items.length); - for (const { - uploadItem: fileOrPath, - fileName, - collectionID, - } of files) { + for (const { uploadItem, fileName, collectionID } of items) { this.abortIfCancelled(); log.info(`Parsing metadata JSON ${fileName}`); - const metadataJSON = await tryParseTakeoutMetadataJSON(fileOrPath); + const metadataJSON = await tryParseTakeoutMetadataJSON( + ensure(uploadItem), + ); if (metadataJSON) { this.parsedMetadataJSONMap.set( getMetadataJSONMapKeyForJSON(collectionID, fileName), @@ -823,7 +821,7 @@ export type UploadableUploadItem = ClusteredUploadItem & { collection: Collection; }; -const splitMetadataAndMediaFiles = ( +const splitMetadataAndMediaItems = ( items: UploadItemWithCollectionIDAndName[], ): [ metadata: UploadItemWithCollectionIDAndName[], @@ -879,13 +877,6 @@ const markUploaded = async (electron: Electron, item: ClusteredUploadItem) => { } }; -/** - * NOTE: a stop gap measure, only meant to be called by code that is running in - * the context of a desktop app initiated upload - */ -export const getFilePathElectron = (file: File | ElectronFile | string) => - typeof file == "string" ? file : (file as ElectronFile).path; - const cancelRemainingUploads = () => ensureElectron().clearPendingUploads(); /** @@ -913,13 +904,13 @@ const clusterLivePhotos = async ( fileName: f.fileName, fileType: fFileType, collectionID: f.collectionID, - fileOrPath: f.uploadItem, + uploadItem: f.uploadItem, }; const ga: PotentialLivePhotoAsset = { fileName: g.fileName, fileType: gFileType, collectionID: g.collectionID, - fileOrPath: g.uploadItem, + uploadItem: g.uploadItem, }; if (await areLivePhotoAssets(fa, ga)) { const [image, video] = @@ -956,7 +947,7 @@ interface PotentialLivePhotoAsset { fileName: string; fileType: FILE_TYPE; collectionID: number; - fileOrPath: File | string; + uploadItem: UploadItem; } const areLivePhotoAssets = async ( @@ -999,8 +990,8 @@ const areLivePhotoAssets = async ( // we use doesn't support stream as a input. const maxAssetSize = 20 * 1024 * 1024; /* 20MB */ - const fSize = await uploadItemSize(f.fileOrPath); - const gSize = await uploadItemSize(g.fileOrPath); + const fSize = await uploadItemSize(f.uploadItem); + const gSize = await uploadItemSize(g.uploadItem); if (fSize > maxAssetSize || gSize > maxAssetSize) { log.info( `Not classifying assets with too large sizes ${[fSize, gSize]} as a live photo`, diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index ed7b16a79..c882d5031 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -6,10 +6,10 @@ * See: [Note: IPC streams]. */ -import type { Electron } from "@/next/types/ipc"; +import type { Electron, ZipEntry } from "@/next/types/ipc"; /** - * Stream the given file from the user's local filesystem. + * Stream the given file or zip entry from the user's local filesystem. * * This only works when we're running in our desktop app since it uses the * "stream://" protocol handler exposed by our custom code in the Node.js layer. @@ -18,8 +18,9 @@ import type { Electron } from "@/next/types/ipc"; * To avoid accidentally invoking it in a non-desktop app context, it requires * the {@link Electron} object as a parameter (even though it doesn't use it). * - * @param path The path on the file on the user's local filesystem whose - * contents we want to stream. + * @param pathOrZipEntry Either the path on the file on the user's local + * filesystem whose contents we want to stream. Or a tuple containing the path + * to a zip file and the name of the entry within it. * * @return A ({@link Response}, size, lastModifiedMs) triple. * @@ -34,16 +35,25 @@ import type { Electron } from "@/next/types/ipc"; */ export const readStream = async ( _: Electron, - path: string, + pathOrZipEntry: string | ZipEntry, ): Promise<{ response: Response; size: number; lastModifiedMs: number }> => { - const req = new Request(`stream://read${path}`, { + let url: URL; + if (typeof pathOrZipEntry == "string") { + url = new URL(`stream://read${pathOrZipEntry}`); + } else { + const [zipPath, entryName] = pathOrZipEntry; + url = new URL(`stream://read${zipPath}`); + url.hash = entryName; + } + + const req = new Request(url, { method: "GET", }); const res = await fetch(req); if (!res.ok) throw new Error( - `Failed to read stream from ${path}: HTTP ${res.status}`, + `Failed to read stream from ${url}: HTTP ${res.status}`, ); const size = readNumericHeader(res, "Content-Length");