beenull/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[desktop] Fix export related IPC - Part 2/x (#1436)

Browse source
This commit is contained in:
Manav Rathi 2024-04-13 21:56:46 +05:30 committed by GitHub
commit b977f982dd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 159 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
web/apps/photos/src/services/export/index.ts
View file

@ -34,22 +34,33 @@ import {
mergeMetadata,
splitFilenameAndExtension,
} from "utils/file";
import {
ENTE_TRASH_FOLDER,
getUniqueCollectionExportName,
getUniqueFileExportName,
} from "utils/native-fs";
import { safeDirectoryName, safeFileName } from "utils/native-fs";
import { getAllLocalCollections } from "../collectionService";
import downloadManager from "../download";
import { getAllLocalFiles } from "../fileService";
import { decodeLivePhoto } from "../livePhotoService";
import { migrateExport } from "./migration";
const EXPORT_RECORD_FILE_NAME = "export_status.json";
/** Name of the JSON file in which we keep the state of the export. */
const exportRecordFileName = "export_status.json";
export const ENTE_EXPORT_DIRECTORY = "ente Photos";
/**
* Name of the top level directory which we create underneath the selected
* directory when the user starts an export to the filesystem.
*/
const exportDirectoryName = "Ente Photos";
export const ENTE_METADATA_FOLDER = "metadata";
/**
* Name of the directory in which we put our metadata when exporting to the
* filesystem.
*/
export const exportMetadataDirectoryName = "metadata";
/**
* Name of the directory in which we keep trash items when deleting files that
* have been exported to the local disk previously.
*/
export const exportTrashDirectoryName = "Trash";
export enum ExportStage {
INIT = 0,
@ -160,7 +171,7 @@ class ExportService {
if (!newRootDir) {
throw Error(CustomError.SELECT_FOLDER_ABORTED);
}
const newExportDir = `${newRootDir}/${ENTE_EXPORT_DIRECTORY}`;
const newExportDir = `${newRootDir}/${exportDirectoryName}`;
await ensureElectron().checkExistsAndCreateDir(newExportDir);
return newExportDir;
} catch (e) {
@ -484,11 +495,10 @@ class ExportService {
const oldCollectionExportName =
collectionIDExportNameMap.get(collection.id);
const oldCollectionExportPath = `${exportFolder}/${oldCollectionExportName}`;
const newCollectionExportName =
await getUniqueCollectionExportName(
exportFolder,
getCollectionUserFacingName(collection),
);
const newCollectionExportName = await safeDirectoryName(
exportFolder,
getCollectionUserFacingName(collection),
);
log.info(
`renaming collection with id ${collection.id} from ${oldCollectionExportName} to ${newCollectionExportName}`,
);
@ -960,7 +970,7 @@ class ExportService {
const exportRecord = await this.getExportRecord(folder);
const newRecord: ExportRecord = { ...exportRecord, ...newData };
await ensureElectron().saveFileToDisk(
`${folder}/${EXPORT_RECORD_FILE_NAME}`,
`${folder}/${exportRecordFileName}`,
JSON.stringify(newRecord, null, 2),
);
return newRecord;
@ -976,7 +986,7 @@ class ExportService {
async getExportRecord(folder: string, retry = true): Promise<ExportRecord> {
try {
await this.verifyExportFolderExists(folder);
const exportRecordJSONPath = `${folder}/${EXPORT_RECORD_FILE_NAME}`;
const exportRecordJSONPath = `${folder}/${exportRecordFileName}`;
if (!(await this.exists(exportRecordJSONPath))) {
return this.createEmptyExportRecord(exportRecordJSONPath);
}
@ -1009,7 +1019,7 @@ class ExportService {
) {
await this.verifyExportFolderExists(exportFolder);
const collectionName = collectionIDNameMap.get(collectionID);
const collectionExportName = await getUniqueCollectionExportName(
const collectionExportName = await safeDirectoryName(
exportFolder,
collectionName,
);
@ -1047,7 +1057,7 @@ class ExportService {
file,
);
} else {
const fileExportName = await getUniqueFileExportName(
const fileExportName = await safeFileName(
collectionExportPath,
file.metadata.title,
);
@ -1086,11 +1096,11 @@ class ExportService {
) {
const fileBlob = await new Response(fileStream).blob();
const livePhoto = await decodeLivePhoto(file, fileBlob);
const imageExportName = await getUniqueFileExportName(
const imageExportName = await safeFileName(
collectionExportPath,
livePhoto.imageNameTitle,
);
const videoExportName = await getUniqueFileExportName(
const videoExportName = await safeFileName(
collectionExportPath,
livePhoto.videoNameTitle,
);
@ -1390,16 +1400,17 @@ const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => {
};
export const getMetadataFolderExportPath = (collectionExportPath: string) =>
`${collectionExportPath}/${ENTE_METADATA_FOLDER}`;
`${collectionExportPath}/${exportMetadataDirectoryName}`;
const getFileMetadataExportPath = (
collectionExportPath: string,
fileExportName: string,
) => `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${fileExportName}.json`;
) =>
`${collectionExportPath}/${exportMetadataDirectoryName}/${fileExportName}.json`;
const getTrashedFileExportPath = async (exportDir: string, path: string) => {
const fileRelativePath = path.replace(`${exportDir}/`, "");
let trashedFilePath = `${exportDir}/${ENTE_TRASH_FOLDER}/${fileRelativePath}`;
let trashedFilePath = `${exportDir}/${exportTrashDirectoryName}/${fileRelativePath}`;
let count = 1;
while (await exportService.exists(trashedFilePath)) {
const trashedFilePathParts = splitFilenameAndExtension(trashedFilePath);
@ -1419,7 +1430,7 @@ const getMetadataFileExportPath = (filePath: string) => {
// extract filename and collection folder path
const filename = filePath.split("/").pop();
const collectionExportPath = filePath.replace(`/${filename}`, "");
return `${collectionExportPath}/${ENTE_METADATA_FOLDER}/${filename}.json`;
return `${collectionExportPath}/${exportMetadataDirectoryName}/${filename}.json`;
};
export const getLivePhotoExportName = (

114
web/apps/photos/src/services/export/migration.ts
View file

@ -26,9 +26,13 @@ import {
getPersonalFiles,
mergeMetadata,
} from "utils/file";
import { sanitizeName } from "utils/native-fs";
import {
ENTE_METADATA_FOLDER,
safeDirectoryName,
safeFileName,
sanitizeFilename,
} from "utils/native-fs";
import {
exportMetadataDirectoryName,
getCollectionIDFromFileUID,
getExportRecordFileUID,
getLivePhotoExportName,
@ -199,7 +203,7 @@ async function migrateCollectionFolders(
collection.id,
collection.name,
);
const newCollectionExportPath = await getUniqueCollectionFolderPath(
const newCollectionExportPath = await safeDirectoryName(
exportDir,
collection.name,
);
@ -228,36 +232,24 @@ async function migrateFiles(
collectionIDPathMap: Map<number, string>,
) {
for (const file of files) {
const oldFileSavePath = getOldFileSavePath(
collectionIDPathMap.get(file.collectionID),
file,
);
const oldFileMetadataSavePath = getOldFileMetadataSavePath(
collectionIDPathMap.get(file.collectionID),
file,
);
const newFileSaveName = await getUniqueFileSaveName(
collectionIDPathMap.get(file.collectionID),
const collectionPath = collectionIDPathMap.get(file.collectionID);
const metadataPath = `${collectionPath}/${exportMetadataDirectoryName}`;
const oldFileName = `${file.id}_${oldSanitizeName(file.metadata.title)}`;
const oldFilePath = `${collectionPath}/${oldFileName}`;
const oldFileMetadataPath = `${metadataPath}/${oldFileName}.json`;
const newFileName = await safeFileName(
collectionPath,
file.metadata.title,
);
const newFilePath = `${collectionPath}/${newFileName}`;
const newFileMetadataPath = `${metadataPath}/${newFileName}.json`;
const newFileSavePath = getFileSavePath(
collectionIDPathMap.get(file.collectionID),
newFileSaveName,
);
if (!(await exportService.exists(oldFilePath))) continue;
const newFileMetadataSavePath = getFileMetadataSavePath(
collectionIDPathMap.get(file.collectionID),
newFileSaveName,
);
if (!(await exportService.exists(oldFileSavePath))) {
continue;
}
await exportService.rename(oldFileSavePath, newFileSavePath);
await exportService.rename(
oldFileMetadataSavePath,
newFileMetadataSavePath,
);
await exportService.rename(oldFilePath, newFilePath);
await exportService.rename(oldFileMetadataPath, newFileMetadataPath);
}
}
@ -498,51 +490,6 @@ const getExportedFiles = (
const oldSanitizeName = (name: string) =>
name.replaceAll("/", "_").replaceAll(" ", "_");
const getUniqueCollectionFolderPath = async (
dir: string,
collectionName: string,
): Promise<string> => {
let collectionFolderPath = `${dir}/${sanitizeName(collectionName)}`;
let count = 1;
while (await exportService.exists(collectionFolderPath)) {
collectionFolderPath = `${dir}/${sanitizeName(
collectionName,
)}(${count})`;
count++;
}
return collectionFolderPath;
};
export const getMetadataFolderPath = (collectionFolderPath: string) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}`;
const getUniqueFileSaveName = async (
collectionPath: string,
filename: string,
) => {
let fileSaveName = sanitizeName(filename);
let count = 1;
while (
await exportService.exists(
getFileSavePath(collectionPath, fileSaveName),
)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileSaveName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileSaveName = `${filenameParts[0]}(${count})`;
}
count++;
}
return fileSaveName;
};
const getFileMetadataSavePath = (
collectionFolderPath: string,
fileSaveName: string,
) => `${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${fileSaveName}.json`;
const getFileSavePath = (collectionFolderPath: string, fileSaveName: string) =>
`${collectionFolderPath}/${fileSaveName}`;
@ -552,32 +499,21 @@ const getOldCollectionFolderPath = (
collectionName: string,
) => `${dir}/${collectionID}_${oldSanitizeName(collectionName)}`;
const getOldFileSavePath = (collectionFolderPath: string, file: EnteFile) =>
`${collectionFolderPath}/${file.id}_${oldSanitizeName(
file.metadata.title,
)}`;
const getOldFileMetadataSavePath = (
collectionFolderPath: string,
file: EnteFile,
) =>
`${collectionFolderPath}/${ENTE_METADATA_FOLDER}/${
file.id
}_${oldSanitizeName(file.metadata.title)}.json`;
const getUniqueFileExportNameForMigration = (
collectionPath: string,
filename: string,
usedFilePaths: Map<string, Set<string>>,
) => {
let fileExportName = sanitizeName(filename);
let fileExportName = sanitizeFilename(filename);
let count = 1;
while (
usedFilePaths
.get(collectionPath)
?.has(getFileSavePath(collectionPath, fileExportName))
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
const filenameParts = splitFilenameAndExtension(
sanitizeFilename(filename),
);
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {

4
web/apps/photos/src/utils/collection/index.ts
View file

@ -44,7 +44,7 @@ import { SetFilesDownloadProgressAttributes } from "types/gallery";
import { SUB_TYPE, VISIBILITY_STATE } from "types/magicMetadata";
import { downloadFilesWithProgress } from "utils/file";
import { isArchivedCollection, updateMagicMetadata } from "utils/magicMetadata";
import { getUniqueCollectionExportName } from "utils/native-fs";
import { safeDirectoryName } from "utils/native-fs";
export enum COLLECTION_OPS_TYPE {
ADD,
@ -169,7 +169,7 @@ async function createCollectionDownloadFolder(
downloadDirPath: string,
collectionName: string,
) {
const collectionDownloadName = await getUniqueCollectionExportName(
const collectionDownloadName = await safeDirectoryName(
downloadDirPath,
collectionName,
);

8
web/apps/photos/src/utils/file/index.ts
View file

@ -52,7 +52,7 @@ import {
import { VISIBILITY_STATE } from "types/magicMetadata";
import { FileTypeInfo } from "types/upload";
import { isArchivedFile, updateMagicMetadata } from "utils/magicMetadata";
import { getUniqueFileExportName } from "utils/native-fs";
import { safeFileName } from "utils/native-fs";
const WAIT_TIME_IMAGE_CONVERSION = 30 * 1000;
@ -812,7 +812,7 @@ async function downloadFileDesktop(
if (file.metadata.fileType === FILE_TYPE.LIVE_PHOTO) {
const fileBlob = await new Response(updatedFileStream).blob();
const livePhoto = await decodeLivePhoto(file, fileBlob);
const imageExportName = await getUniqueFileExportName(
const imageExportName = await safeFileName(
downloadPath,
livePhoto.imageNameTitle,
);
@ -822,7 +822,7 @@ async function downloadFileDesktop(
imageStream,
);
try {
const videoExportName = await getUniqueFileExportName(
const videoExportName = await safeFileName(
downloadPath,
livePhoto.videoNameTitle,
);
@ -836,7 +836,7 @@ async function downloadFileDesktop(
throw e;
}
} else {
const fileExportName = await getUniqueFileExportName(
const fileExportName = await safeFileName(
downloadPath,
file.metadata.title,
);

98
web/apps/photos/src/utils/native-fs.ts
View file

@ -1,44 +1,84 @@
/**
* @file Native filesystem access using custom Node.js functionality provided by
* our desktop app.
*
* Precondition: Unless mentioned otherwise, the functions in these file only
* work when we are running in our desktop app.
*/
import { ensureElectron } from "@/next/electron";
import { nameAndExtension } from "@/next/file";
import sanitize from "sanitize-filename";
import exportService from "services/export";
import { splitFilenameAndExtension } from "utils/file";
import {
exportMetadataDirectoryName,
exportTrashDirectoryName,
} from "services/export";
export const ENTE_TRASH_FOLDER = "Trash";
/**
* Sanitize string for use as file or directory name.
*
* Return a string suitable for use as a file or directory name by replacing
* directory separators and invalid characters in the input string {@link s}
* with "_".
*/
export const sanitizeFilename = (s: string) =>
sanitize(s, { replacement: "_" });
export const sanitizeName = (name: string) =>
sanitize(name, { replacement: "_" });
export const getUniqueCollectionExportName = async (
dir: string,
collectionName: string,
/**
* Return a new sanitized and unique directory name based on {@link name} that
* is not the same as any existing item in the given {@link directoryPath}.
*
* We also ensure we don't return names which might collide with our own special
* directories.
*
* This function only works when we are running inside an electron app (since it
* requires permissionless access to the native filesystem to find a new
* filename that doesn't conflict with any existing items).
*
* See also: {@link safeDirectoryName}
*/
export const safeDirectoryName = async (
directoryPath: string,
name: string,
): Promise<string> => {
let collectionExportName = sanitizeName(collectionName);
const specialDirectoryNames = [
exportTrashDirectoryName,
exportMetadataDirectoryName,
];
let result = sanitizeFilename(name);
let count = 1;
while (
(await exportService.exists(`${dir}/${collectionExportName}`)) ||
collectionExportName === ENTE_TRASH_FOLDER
(await exists(`${directoryPath}/${result}`)) ||
specialDirectoryNames.includes(result)
) {
collectionExportName = `${sanitizeName(collectionName)}(${count})`;
result = `${sanitizeFilename(name)}(${count})`;
count++;
}
return collectionExportName;
return result;
};
export const getUniqueFileExportName = async (
collectionExportPath: string,
filename: string,
) => {
let fileExportName = sanitizeName(filename);
/**
* Return a new sanitized and unique file name based on {@link name} that is not
* the same as any existing item in the given {@link directoryPath}.
*
* This function only works when we are running inside an electron app.
* @see {@link safeDirectoryName}.
*/
export const safeFileName = async (directoryPath: string, name: string) => {
let result = sanitizeFilename(name);
let count = 1;
while (
await exportService.exists(`${collectionExportPath}/${fileExportName}`)
) {
const filenameParts = splitFilenameAndExtension(sanitizeName(filename));
if (filenameParts[1]) {
fileExportName = `${filenameParts[0]}(${count}).${filenameParts[1]}`;
} else {
fileExportName = `${filenameParts[0]}(${count})`;
}
while (await exists(`${directoryPath}/${result}`)) {
const [fn, ext] = nameAndExtension(sanitizeFilename(name));
if (ext) result = `${fn}(${count}).${ext}`;
else result = `${fn}(${count})`;
count++;
}
return fileExportName;
return result;
};
/**
* Return true if an item exists an the given {@link path} on the user's local
* filesystem.
*/
export const exists = (path: string) => ensureElectron().fs.exists(path);

4
web/apps/photos/src/utils/upload/index.ts
View file

@ -5,7 +5,7 @@ import {
PICKED_UPLOAD_TYPE,
} from "constants/upload";
import isElectron from "is-electron";
import { ENTE_METADATA_FOLDER } from "services/export";
import { exportMetadataDirectoryName } from "services/export";
import { EnteFile } from "types/file";
import {
ElectronFile,
@ -175,7 +175,7 @@ export function groupFilesBasedOnParentFolder(
// For Eg,For FileList -> [a/x.png, a/metadata/x.png.json]
// they will both we grouped into the collection "a"
// This is cluster the metadata json files in the same collection as the file it is for
if (folderPath.endsWith(ENTE_METADATA_FOLDER)) {
if (folderPath.endsWith(exportMetadataDirectoryName)) {
folderPath = folderPath.substring(0, folderPath.lastIndexOf("/"));
}
const folderName = folderPath.substring(

7
web/docs/dependencies.md
View file

@ -130,3 +130,10 @@ For some of our newer code, we have started to use [Vite](https://vitejs.dev).
It is more lower level than Next, but the bells and whistles it doesn't have are
the bells and whistles (and the accompanying complexity) that we don't need in
some cases.
## Photos
### Misc
- "sanitize-filename" is for converting arbitrary strings into strings that
are suitable for being used as filenames.

14
web/packages/next/file.ts
View file

@ -1,5 +1,19 @@
import type { ElectronFile } from "./types/file";
/**
* Split a filename into its components - the name itself, and the extension (if
* any) - returning both. The dot is not included in either.
*
* For example, `foo-bar.png` will be split into ["foo-bar", "png"].
*/
export const nameAndExtension = (
fileName: string,
): [string, string | undefined] => {
const i = fileName.lastIndexOf(".");
if (i == -1) return [fileName, undefined];
else return [fileName.slice(0, i), fileName.slice(i + 1)];
};
export function getFileNameSize(file: File | ElectronFile) {
return `${file.name}_${convertBytesToHumanReadable(file.size)}`;
}

2
web/packages/next/types/ipc.ts
View file

@ -80,7 +80,7 @@ export interface Electron {
*
* If no such key is found, return `undefined`.
*
* @see {@link saveEncryptionKey}.
* See also: {@link saveEncryptionKey}.
*/
encryptionKey: () => Promise<string | undefined>;