Merge pull request #444 from ente-io/recover-failed-imports

desktop upload
This commit is contained in:
Abhinav Kumar 2022-04-19 14:39:17 +05:30 committed by GitHub
commit 19642555a3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 768 additions and 202 deletions

View file

@ -86,6 +86,7 @@
"husky": "^7.0.1",
"lint-staged": "^11.1.2",
"prettier": "2.3.2",
"react-icons": "^4.3.1",
"typescript": "^4.1.3"
},
"standard": {

View file

@ -17,7 +17,7 @@ const Wrapper = styled.div`
}
`;
export default function EmptyScreen({ openFileUploader }) {
export default function EmptyScreen({ openUploader }) {
const deduplicateContext = useContext(DeduplicateContext);
return (
<Wrapper>
@ -37,7 +37,7 @@ export default function EmptyScreen({ openFileUploader }) {
</div>
<Button
variant="outline-success"
onClick={openFileUploader}
onClick={openUploader}
style={{
marginTop: '32px',
paddingLeft: '32px',

View file

@ -53,7 +53,7 @@ interface Props {
) => void;
selected: SelectedState;
isFirstLoad?;
openFileUploader?;
openUploader?;
isInSearchMode?: boolean;
search?: Search;
setSearchStats?: setSearchStats;
@ -77,7 +77,7 @@ const PhotoFrame = ({
setSelected,
selected,
isFirstLoad,
openFileUploader,
openUploader,
isInSearchMode,
search,
setSearchStats,
@ -548,7 +548,7 @@ const PhotoFrame = ({
return (
<>
{!isFirstLoad && files.length === 0 && !isInSearchMode ? (
<EmptyScreen openFileUploader={openFileUploader} />
<EmptyScreen openUploader={openUploader} />
) : (
<Container>
<AutoSizer>

View file

@ -0,0 +1,23 @@
import React from 'react';
export default function FileUploadIcon(props) {
return (
<svg
width={props.width}
height={props.height}
viewBox={props.viewBox}
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path
d="M28 25.3333V6.66667C28 5.2 26.8 4 25.3333 4H6.66667C5.2 4 4 5.2 4 6.66667V25.3333C4 26.8 5.2 28 6.66667 28H25.3333C26.8 28 28 26.8 28 25.3333ZM11.8667 18.64L14.6667 22.0133L18.8 16.6933C19.0667 16.3467 19.6 16.3467 19.8667 16.7067L24.5467 22.9467C24.88 23.3867 24.56 24.0133 24.0133 24.0133H8.02667C7.46667 24.0133 7.16 23.3733 7.50667 22.9333L10.8267 18.6667C11.08 18.32 11.5867 18.3067 11.8667 18.64V18.64Z"
fill="black"
/>
</svg>
);
}
FileUploadIcon.defaultProps = {
height: 32,
width: 32,
viewBox: '0 0 32 32',
};

View file

@ -0,0 +1,27 @@
import React from 'react';
export default function FolderUploadIcon(props) {
return (
<svg
width={props.width}
height={props.height}
viewBox={props.viewBox}
fill="none"
xmlns="http://www.w3.org/2000/svg">
<path
d="M24.333 23.3333H2.99967V7.33333C2.99967 6.6 2.39967 6 1.66634 6C0.933008 6 0.333008 6.6 0.333008 7.33333V23.3333C0.333008 24.8 1.53301 26 2.99967 26H24.333C25.0663 26 25.6663 25.4 25.6663 24.6667C25.6663 23.9333 25.0663 23.3333 24.333 23.3333Z"
fill="black"
/>
<path
d="M26.9993 3.33366H17.666L15.786 1.45366C15.2793 0.946992 14.5993 0.666992 13.8927 0.666992H8.33268C6.86602 0.666992 5.67935 1.86699 5.67935 3.33366L5.66602 18.0003C5.66602 19.467 6.86602 20.667 8.33268 20.667H26.9993C28.466 20.667 29.666 19.467 29.666 18.0003V6.00033C29.666 4.53366 28.466 3.33366 26.9993 3.33366ZM22.9993 15.3337H12.3327C11.786 15.3337 11.466 14.707 11.7993 14.267L13.6393 11.827C13.906 11.467 14.4393 11.467 14.706 11.827L16.3327 14.0003L19.2927 10.0403C19.5593 9.68033 20.0927 9.68033 20.3594 10.0403L23.5327 14.267C23.866 14.707 23.546 15.3337 22.9993 15.3337Z"
fill="black"
/>
</svg>
);
}
FolderUploadIcon.defaultProps = {
height: 32,
width: 32,
viewBox: '0 0 32 32',
};

View file

@ -1,11 +1,11 @@
import React, { useContext, useEffect, useState } from 'react';
import React, { useContext, useEffect, useRef, useState } from 'react';
import { syncCollections, createAlbum } from 'services/collectionService';
import constants from 'utils/strings/constants';
import { SetDialogMessage } from 'components/MessageDialog';
import UploadProgress from './UploadProgress';
import ChoiceModal from './ChoiceModal';
import UploadStrategyChoiceModal from './UploadStrategyChoiceModal';
import { SetCollectionNamerAttributes } from './CollectionNamer';
import { SetCollectionSelectorAttributes } from './CollectionSelector';
import { GalleryContext } from 'pages/gallery';
@ -14,12 +14,15 @@ import { logError } from 'utils/sentry';
import { FileRejection } from 'react-dropzone';
import UploadManager from 'services/upload/uploadManager';
import uploadManager from 'services/upload/uploadManager';
import ImportService from 'services/importService';
import isElectron from 'is-electron';
import { METADATA_FOLDER_NAME } from 'constants/export';
import { getUserFacingErrorMessage } from 'utils/error';
import { Collection } from 'types/collection';
import { SetLoading, SetFiles } from 'types/gallery';
import { FileUploadResults, UPLOAD_STAGES } from 'constants/upload';
import { FileWithCollection } from 'types/upload';
import { ElectronFile, FileWithCollection } from 'types/upload';
import UploadTypeChoiceModal from './UploadTypeChoiceModal';
const FIRST_ALBUM_NAME = 'My First Album';
@ -37,6 +40,10 @@ interface Props {
fileRejections: FileRejection[];
setFiles: SetFiles;
isFirstUpload: boolean;
electronFiles: ElectronFile[];
setElectronFiles: (files: ElectronFile[]) => void;
showUploadTypeChoiceModal: boolean;
setShowUploadTypeChoiceModal: (open: boolean) => void;
}
enum UPLOAD_STRATEGY {
@ -44,6 +51,11 @@ enum UPLOAD_STRATEGY {
COLLECTION_PER_FOLDER,
}
enum DESKTOP_UPLOAD_TYPE {
FILES,
FOLDERS,
}
interface AnalysisResult {
suggestedCollectionName: string;
multipleFolders: boolean;
@ -71,6 +83,11 @@ export default function Upload(props: Props) {
const appContext = useContext(AppContext);
const galleryContext = useContext(GalleryContext);
const toUploadFiles = useRef<File[] | ElectronFile[]>(null);
const isPendingDesktopUpload = useRef(false);
const pendingDesktopUploadCollectionName = useRef<string>('');
const desktopUploadType = useRef<DESKTOP_UPLOAD_TYPE>(null);
useEffect(() => {
UploadManager.initUploader(
{
@ -84,24 +101,43 @@ export default function Upload(props: Props) {
},
props.setFiles
);
if (isElectron()) {
ImportService.getPendingUploads().then(
({ files: electronFiles, collectionName }) => {
resumeDesktopUpload(electronFiles, collectionName);
}
);
}
}, []);
useEffect(() => {
if (
props.acceptedFiles?.length > 0 ||
appContext.sharedFiles?.length > 0
appContext.sharedFiles?.length > 0 ||
props.electronFiles?.length > 0
) {
props.setLoading(true);
let analysisResult: AnalysisResult;
if (props.acceptedFiles?.length > 0) {
// File selection by drag and drop or selection of file.
if (
props.acceptedFiles?.length > 0 ||
props.electronFiles?.length > 0
) {
if (props.acceptedFiles?.length > 0) {
// File selection by drag and drop or selection of file.
toUploadFiles.current = props.acceptedFiles;
} else {
// File selection from desktop app
toUploadFiles.current = props.electronFiles;
}
analysisResult = analyseUploadFiles();
if (analysisResult) {
setAnalysisResult(analysisResult);
}
} else {
props.acceptedFiles = appContext.sharedFiles;
} else if (appContext.sharedFiles.length > 0) {
toUploadFiles.current = appContext.sharedFiles;
}
handleCollectionCreationAndUpload(
analysisResult,
@ -109,7 +145,7 @@ export default function Upload(props: Props) {
);
props.setLoading(false);
}
}, [props.acceptedFiles, appContext.sharedFiles]);
}, [props.acceptedFiles, appContext.sharedFiles, props.electronFiles]);
const uploadInit = function () {
setUploadStage(UPLOAD_STAGES.START);
@ -121,11 +157,28 @@ export default function Upload(props: Props) {
setProgressView(true);
};
const resumeDesktopUpload = async (
electronFiles: ElectronFile[],
collectionName: string
) => {
if (electronFiles && electronFiles?.length > 0) {
isPendingDesktopUpload.current = true;
pendingDesktopUploadCollectionName.current = collectionName;
props.setElectronFiles(electronFiles);
}
};
function analyseUploadFiles(): AnalysisResult {
if (props.acceptedFiles.length === 0) {
if (toUploadFiles.current.length === 0) {
return null;
}
const paths: string[] = props.acceptedFiles.map((file) => file['path']);
if (desktopUploadType.current === DESKTOP_UPLOAD_TYPE.FILES) {
desktopUploadType.current = null;
return { suggestedCollectionName: '', multipleFolders: false };
}
const paths: string[] = toUploadFiles.current.map(
(file) => file['path']
);
const getCharCount = (str: string) => (str.match(/\//g) ?? []).length;
paths.sort((path1, path2) => getCharCount(path1) - getCharCount(path2));
const firstPath = paths[0];
@ -153,8 +206,8 @@ export default function Upload(props: Props) {
};
}
function getCollectionWiseFiles() {
const collectionWiseFiles = new Map<string, File[]>();
for (const file of props.acceptedFiles) {
const collectionWiseFiles = new Map<string, (File | ElectronFile)[]>();
for (const file of toUploadFiles.current) {
const filePath = file['path'] as string;
let folderPath = filePath.substr(0, filePath.lastIndexOf('/'));
@ -172,16 +225,16 @@ export default function Upload(props: Props) {
return collectionWiseFiles;
}
const uploadFilesToExistingCollection = async (collection) => {
const uploadFilesToExistingCollection = async (collection: Collection) => {
try {
uploadInit();
const filesWithCollectionToUpload: FileWithCollection[] =
props.acceptedFiles.map((file, index) => ({
toUploadFiles.current.map((file, index) => ({
file,
localID: index,
collectionID: collection.id,
}));
await uploadFiles(filesWithCollectionToUpload);
await uploadFiles(filesWithCollectionToUpload, [collection]);
} catch (e) {
logError(e, 'Failed to upload files to existing collections');
}
@ -196,9 +249,12 @@ export default function Upload(props: Props) {
const filesWithCollectionToUpload: FileWithCollection[] = [];
const collections: Collection[] = [];
let collectionWiseFiles = new Map<string, File[]>();
let collectionWiseFiles = new Map<
string,
(File | ElectronFile)[]
>();
if (strategy === UPLOAD_STRATEGY.SINGLE_COLLECTION) {
collectionWiseFiles.set(collectionName, props.acceptedFiles);
collectionWiseFiles.set(collectionName, toUploadFiles.current);
} else {
collectionWiseFiles = getCollectionWiseFiles();
}
@ -239,12 +295,18 @@ export default function Upload(props: Props) {
const uploadFiles = async (
filesWithCollectionToUpload: FileWithCollection[],
collections?: Collection[]
collections: Collection[]
) => {
try {
props.setUploadInProgress(true);
props.closeCollectionSelector();
await props.syncWithRemote(true, true);
if (isElectron()) {
await ImportService.setToUploadFiles(
filesWithCollectionToUpload,
collections
);
}
await uploadManager.queueFilesForUpload(
filesWithCollectionToUpload,
collections
@ -306,6 +368,19 @@ export default function Upload(props: Props) {
analysisResult: AnalysisResult,
isFirstUpload: boolean
) => {
if (isPendingDesktopUpload.current) {
isPendingDesktopUpload.current = false;
if (pendingDesktopUploadCollectionName.current) {
uploadToSingleNewCollection(
pendingDesktopUploadCollectionName.current
);
} else {
uploadFilesToNewCollections(
UPLOAD_STRATEGY.COLLECTION_PER_FOLDER
);
}
return;
}
if (isFirstUpload && !analysisResult.suggestedCollectionName) {
analysisResult.suggestedCollectionName = FIRST_ALBUM_NAME;
}
@ -324,10 +399,31 @@ export default function Upload(props: Props) {
title: constants.UPLOAD_TO_COLLECTION,
});
};
const handleDesktopUploadTypes = async (type: DESKTOP_UPLOAD_TYPE) => {
let files: ElectronFile[];
desktopUploadType.current = type;
if (type === DESKTOP_UPLOAD_TYPE.FILES) {
files = await ImportService.showUploadFilesDialog();
} else {
files = await ImportService.showUploadDirsDialog();
}
props.setElectronFiles(files);
props.setShowUploadTypeChoiceModal(false);
};
const cancelUploads = async () => {
setProgressView(false);
UploadManager.cancelRemainingUploads();
if (isElectron()) {
ImportService.updatePendingUploads([]);
}
await props.setUploadInProgress(false);
await props.syncWithRemote();
};
return (
<>
<ChoiceModal
<UploadStrategyChoiceModal
show={choiceModalView}
onHide={() => setChoiceModalView(false)}
uploadToSingleCollection={() =>
@ -341,6 +437,16 @@ export default function Upload(props: Props) {
)
}
/>
<UploadTypeChoiceModal
show={props.showUploadTypeChoiceModal}
onHide={() => props.setShowUploadTypeChoiceModal(false)}
uploadFiles={() =>
handleDesktopUploadTypes(DESKTOP_UPLOAD_TYPE.FILES)
}
uploadFolders={() =>
handleDesktopUploadTypes(DESKTOP_UPLOAD_TYPE.FOLDERS)
}
/>
<UploadProgress
now={percentComplete}
filenames={filenames}
@ -353,6 +459,7 @@ export default function Upload(props: Props) {
retryFailed={retryFailed}
fileRejections={props.fileRejections}
uploadResult={uploadResult}
cancelUploads={cancelUploads}
/>
</>
);

View file

@ -14,9 +14,9 @@ const Wrapper = styled.div<{ isDisabled: boolean }>`
cursor: pointer;
opacity: ${(props) => (props.isDisabled ? 0 : 1)};
`;
function UploadButton({ openFileUploader, isFirstFetch }) {
function UploadButton({ isFirstFetch, openUploader }) {
return (
<Wrapper onClick={openFileUploader} isDisabled={isFirstFetch}>
<Wrapper onClick={openUploader} isDisabled={isFirstFetch}>
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"

View file

@ -1,6 +1,6 @@
import ExpandLess from 'components/icons/ExpandLess';
import ExpandMore from 'components/icons/ExpandMore';
import React, { useState } from 'react';
import React, { useContext, useState } from 'react';
import { Accordion, Button, Modal, ProgressBar } from 'react-bootstrap';
import { FileRejection } from 'react-dropzone';
@ -10,6 +10,7 @@ import constants from 'utils/strings/constants';
import { ButtonVariant, getVariantColor } from './LinkButton';
import { FileUploadResults, UPLOAD_STAGES } from 'constants/upload';
import FileList from 'components/FileList';
import { AppContext } from 'pages/_app';
interface Props {
fileCounter;
uploadStage;
@ -22,6 +23,7 @@ interface Props {
fileRejections: FileRejection[];
uploadResult: Map<number, FileUploadResults>;
hasLivePhotos: boolean;
cancelUploads: () => void;
}
interface FileProgresses {
fileID: number;
@ -167,6 +169,8 @@ const InProgressSection = (props: InProgressProps) => {
};
export default function UploadProgress(props: Props) {
const appContext = useContext(AppContext);
const fileProgressStatuses = [] as FileProgresses[];
const fileUploadResultMap = new Map<FileUploadResults, number[]>();
let filesNotUploaded = false;
@ -196,138 +200,162 @@ export default function UploadProgress(props: Props) {
sectionInfo = constants.LIVE_PHOTOS_DETECTED();
}
function handleHideModal(): () => void {
return props.uploadStage !== UPLOAD_STAGES.FINISH
? () => {
appContext.setDialogMessage({
title: constants.STOP_UPLOADS_HEADER,
content: constants.STOP_ALL_UPLOADS_MESSAGE,
proceed: {
text: constants.YES_STOP_UPLOADS,
variant: 'danger',
action: props.cancelUploads,
},
close: {
text: constants.NO,
variant: 'secondary',
action: () => {},
},
});
}
: props.closeModal;
}
return (
<Modal
show={props.show}
onHide={
props.uploadStage !== UPLOAD_STAGES.FINISH
? () => null
: props.closeModal
}
aria-labelledby="contained-modal-title-vcenter"
centered
backdrop={fileProgressStatuses?.length !== 0 ? 'static' : true}>
<Modal.Header
style={{
display: 'flex',
justifyContent: 'center',
textAlign: 'center',
borderBottom: 'none',
paddingTop: '30px',
paddingBottom: '0px',
}}
closeButton={props.uploadStage === UPLOAD_STAGES.FINISH}>
<h4 style={{ width: '100%' }}>
{props.uploadStage === UPLOAD_STAGES.UPLOADING
? constants.UPLOAD[props.uploadStage](props.fileCounter)
: constants.UPLOAD[props.uploadStage]}
</h4>
</Modal.Header>
<Modal.Body>
{(props.uploadStage ===
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES ||
props.uploadStage === UPLOAD_STAGES.EXTRACTING_METADATA ||
props.uploadStage === UPLOAD_STAGES.UPLOADING) && (
<ProgressBar
now={props.now}
animated
variant="upload-progress-bar"
/>
)}
{props.uploadStage === UPLOAD_STAGES.UPLOADING && (
<InProgressSection
<>
<Modal
show={props.show}
onHide={handleHideModal()}
aria-labelledby="contained-modal-title-vcenter"
centered
backdrop={fileProgressStatuses?.length !== 0 ? 'static' : true}>
<Modal.Header
style={{
display: 'flex',
justifyContent: 'center',
textAlign: 'center',
borderBottom: 'none',
paddingTop: '30px',
paddingBottom: '0px',
}}
closeButton={true}>
<h4 style={{ width: '100%' }}>
{props.uploadStage === UPLOAD_STAGES.UPLOADING
? constants.UPLOAD[props.uploadStage](
props.fileCounter
)
: constants.UPLOAD[props.uploadStage]}
</h4>
</Modal.Header>
<Modal.Body>
{(props.uploadStage ===
UPLOAD_STAGES.READING_GOOGLE_METADATA_FILES ||
props.uploadStage ===
UPLOAD_STAGES.EXTRACTING_METADATA ||
props.uploadStage === UPLOAD_STAGES.UPLOADING) && (
<ProgressBar
now={props.now}
animated
variant="upload-progress-bar"
/>
)}
{props.uploadStage === UPLOAD_STAGES.UPLOADING && (
<InProgressSection
filenames={props.filenames}
fileProgressStatuses={fileProgressStatuses}
sectionTitle={constants.INPROGRESS_UPLOADS}
sectionInfo={sectionInfo}
/>
)}
<ResultSection
filenames={props.filenames}
fileProgressStatuses={fileProgressStatuses}
sectionTitle={constants.INPROGRESS_UPLOADS}
sectionInfo={sectionInfo}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.UPLOADED}
sectionTitle={constants.SUCCESSFUL_UPLOADS}
/>
)}
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.UPLOADED}
sectionTitle={constants.SUCCESSFUL_UPLOADS}
/>
{props.uploadStage === UPLOAD_STAGES.FINISH &&
filesNotUploaded && (
<NotUploadSectionHeader>
{constants.FILE_NOT_UPLOADED_LIST}
</NotUploadSectionHeader>
)}
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.BLOCKED}
sectionTitle={constants.BLOCKED_UPLOADS}
sectionInfo={constants.ETAGS_BLOCKED(
DESKTOP_APP_DOWNLOAD_URL
)}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.FAILED}
sectionTitle={constants.FAILED_UPLOADS}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.ALREADY_UPLOADED}
sectionTitle={constants.SKIPPED_FILES}
sectionInfo={constants.SKIPPED_INFO}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={
FileUploadResults.LARGER_THAN_AVAILABLE_STORAGE
}
sectionTitle={
constants.LARGER_THAN_AVAILABLE_STORAGE_UPLOADS
}
sectionInfo={constants.LARGER_THAN_AVAILABLE_STORAGE_INFO}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.UNSUPPORTED}
sectionTitle={constants.UNSUPPORTED_FILES}
sectionInfo={constants.UNSUPPORTED_INFO}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.TOO_LARGE}
sectionTitle={constants.TOO_LARGE_UPLOADS}
sectionInfo={constants.TOO_LARGE_INFO}
/>
</Modal.Body>
{props.uploadStage === UPLOAD_STAGES.FINISH && (
<Modal.Footer style={{ border: 'none' }}>
{props.uploadStage === UPLOAD_STAGES.FINISH &&
(fileUploadResultMap?.get(FileUploadResults.FAILED)
?.length > 0 ||
fileUploadResultMap?.get(FileUploadResults.BLOCKED)
?.length > 0 ? (
<Button
variant="outline-success"
style={{ width: '100%' }}
onClick={props.retryFailed}>
{constants.RETRY_FAILED}
</Button>
) : (
<Button
variant="outline-secondary"
style={{ width: '100%' }}
onClick={props.closeModal}>
{constants.CLOSE}
</Button>
))}
</Modal.Footer>
)}
</Modal>
filesNotUploaded && (
<NotUploadSectionHeader>
{constants.FILE_NOT_UPLOADED_LIST}
</NotUploadSectionHeader>
)}
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.BLOCKED}
sectionTitle={constants.BLOCKED_UPLOADS}
sectionInfo={constants.ETAGS_BLOCKED(
DESKTOP_APP_DOWNLOAD_URL
)}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.FAILED}
sectionTitle={constants.FAILED_UPLOADS}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.ALREADY_UPLOADED}
sectionTitle={constants.SKIPPED_FILES}
sectionInfo={constants.SKIPPED_INFO}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={
FileUploadResults.LARGER_THAN_AVAILABLE_STORAGE
}
sectionTitle={
constants.LARGER_THAN_AVAILABLE_STORAGE_UPLOADS
}
sectionInfo={
constants.LARGER_THAN_AVAILABLE_STORAGE_INFO
}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.UNSUPPORTED}
sectionTitle={constants.UNSUPPORTED_FILES}
sectionInfo={constants.UNSUPPORTED_INFO}
/>
<ResultSection
filenames={props.filenames}
fileUploadResultMap={fileUploadResultMap}
fileUploadResult={FileUploadResults.TOO_LARGE}
sectionTitle={constants.TOO_LARGE_UPLOADS}
sectionInfo={constants.TOO_LARGE_INFO}
/>
</Modal.Body>
{props.uploadStage === UPLOAD_STAGES.FINISH && (
<Modal.Footer style={{ border: 'none' }}>
{props.uploadStage === UPLOAD_STAGES.FINISH &&
(fileUploadResultMap?.get(FileUploadResults.FAILED)
?.length > 0 ||
fileUploadResultMap?.get(FileUploadResults.BLOCKED)
?.length > 0 ? (
<Button
variant="outline-success"
style={{ width: '100%' }}
onClick={props.retryFailed}>
{constants.RETRY_FAILED}
</Button>
) : (
<Button
variant="outline-secondary"
style={{ width: '100%' }}
onClick={props.closeModal}>
{constants.CLOSE}
</Button>
))}
</Modal.Footer>
)}
</Modal>
</>
);
}

View file

@ -9,7 +9,7 @@ interface Props {
onHide: () => void;
uploadToSingleCollection: () => void;
}
function ChoiceModal({
function UploadStrategyChoiceModal({
uploadToMultipleCollection,
uploadToSingleCollection,
...props
@ -72,4 +72,4 @@ function ChoiceModal({
</MessageDialog>
);
}
export default ChoiceModal;
export default UploadStrategyChoiceModal;

View file

@ -0,0 +1,90 @@
import { Modal, Button, Container, Row } from 'react-bootstrap';
import React from 'react';
import constants from 'utils/strings/constants';
import { IoIosArrowForward, IoMdClose } from 'react-icons/io';
import FileUploadIcon from 'components/icons/FileUploadIcon';
import FolderUploadIcon from 'components/icons/FolderUploadIcon';
export default function UploadTypeChoiceModal({
onHide,
show,
uploadFiles,
uploadFolders,
}) {
return (
<Modal
show={show}
aria-labelledby="contained-modal-title-vcenter"
centered
dialogClassName="file-type-choice-modal">
<Modal.Header
onHide={onHide}
style={{
borderBottom: 'none',
height: '4em',
}}>
<Modal.Title
id="contained-modal-title-vcenter"
style={{
fontSize: '1.8em',
marginLeft: '5%',
color: 'white',
}}>
<b>{constants.CHOOSE_UPLOAD_TYPE}</b>
</Modal.Title>
<IoMdClose
size={30}
onClick={onHide}
style={{ cursor: 'pointer' }}
/>
</Modal.Header>
<Modal.Body
style={{
height: '10em',
}}>
<Container>
<Row className="justify-content-center py-2">
<Button
variant="light"
onClick={uploadFiles}
style={{ width: '90%', height: '3em' }}>
<Container>
<Row>
<div>
<FileUploadIcon />
<b className="ml-2">
{constants.UPLOAD_FILES}
</b>
</div>
<div className="ml-auto d-flex align-items-center">
<IoIosArrowForward />
</div>
</Row>
</Container>
</Button>
</Row>
<Row className="justify-content-center py-2">
<Button
variant="light"
onClick={uploadFolders}
style={{ width: '90%', height: '3em' }}>
<Container>
<Row>
<div>
<FolderUploadIcon />
<b className="ml-2">
{constants.UPLOAD_DIRS}
</b>
</div>
<div className="ml-auto d-flex align-items-center">
<IoIosArrowForward />
</div>
</Row>
</Container>
</Button>
</Row>
</Container>
</Modal.Body>
</Modal>
);
}

View file

@ -48,3 +48,5 @@ export const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = {
location: NULL_LOCATION,
creationTime: null,
};
export const A_SEC_IN_MICROSECONDS = 1e6;

View file

@ -501,6 +501,13 @@ const GlobalStyles = createGlobalStyle`
.form-check-input:hover, .form-check-label :hover{
cursor:pointer;
}
@media (min-width: 450px) {
.file-type-choice-modal{
width: 25em;
}
}
.manageLinkHeader:hover{
color:#bbb;
}

View file

@ -104,6 +104,8 @@ import {
import Collections from 'components/pages/gallery/Collections';
import { VISIBILITY_STATE } from 'types/magicMetadata';
import ToastNotification from 'components/ToastNotification';
import { ElectronFile } from 'types/upload';
import importService from 'services/importService';
export const DeadCenter = styled.div`
flex: 1;
@ -201,6 +203,10 @@ export default function Gallery() {
const clearNotificationAttributes = () => setNotificationAttributes(null);
const [electronFiles, setElectronFiles] = useState<ElectronFile[]>(null);
const [showUploadTypeChoiceModal, setShowUploadTypeChoiceModal] =
useState(false);
useEffect(() => {
const key = getKey(SESSION_KEYS.ENCRYPTION_KEY);
if (!key) {
@ -547,6 +553,14 @@ export default function Gallery() {
finishLoading();
};
const openUploader = () => {
if (importService.checkAllElectronAPIsExists()) {
setShowUploadTypeChoiceModal(true);
} else {
openFileUploader();
}
};
return (
<GalleryContext.Provider
value={{
@ -645,6 +659,10 @@ export default function Gallery() {
fileRejections={fileRejections}
setFiles={setFiles}
isFirstUpload={collectionsAndTheirLatestFile?.length === 0}
electronFiles={electronFiles}
setElectronFiles={setElectronFiles}
showUploadTypeChoiceModal={showUploadTypeChoiceModal}
setShowUploadTypeChoiceModal={setShowUploadTypeChoiceModal}
/>
<Sidebar
collections={collections}
@ -653,7 +671,7 @@ export default function Gallery() {
/>
<UploadButton
isFirstFetch={isFirstFetch}
openFileUploader={openFileUploader}
openUploader={openUploader}
/>
<PhotoFrame
files={files}
@ -664,7 +682,7 @@ export default function Gallery() {
setSelected={setSelected}
selected={selected}
isFirstLoad={isFirstLoad}
openFileUploader={openFileUploader}
openUploader={openUploader}
isInSearchMode={isInSearchMode}
search={search}
setSearchStats={setSearchStats}

View file

@ -286,7 +286,7 @@ export default function PublicCollectionGallery() {
setSelected={() => null}
selected={{ count: 0, collectionID: null }}
isFirstLoad={true}
openFileUploader={() => null}
openUploader={() => null}
isInSearchMode={false}
search={{}}
setSearchStats={() => null}

View file

@ -0,0 +1,97 @@
import { Collection } from 'types/collection';
import { ElectronFile, FileWithCollection } from 'types/upload';
import { runningInBrowser } from 'utils/common';
import { logError } from 'utils/sentry';
interface PendingUploads {
files: ElectronFile[];
collectionName: string;
}
class ImportService {
ElectronAPIs: any;
private allElectronAPIsExist: boolean = false;
constructor() {
this.ElectronAPIs = runningInBrowser() && window['ElectronAPIs'];
this.allElectronAPIsExist = !!this.ElectronAPIs?.getPendingUploads;
}
checkAllElectronAPIsExists = () => this.allElectronAPIsExist;
async showUploadFilesDialog(): Promise<ElectronFile[]> {
if (this.allElectronAPIsExist) {
return this.ElectronAPIs.showUploadFilesDialog();
}
}
async showUploadDirsDialog(): Promise<ElectronFile[]> {
if (this.allElectronAPIsExist) {
return this.ElectronAPIs.showUploadDirsDialog();
}
}
async getPendingUploads(): Promise<PendingUploads> {
try {
if (this.allElectronAPIsExist) {
const pendingUploads =
(await this.ElectronAPIs.getPendingUploads()) as PendingUploads;
return pendingUploads;
}
} catch (e) {
logError(e, 'failed to getPendingUploads ');
return { files: [], collectionName: null };
}
}
async setToUploadFiles(
files: FileWithCollection[],
collections: Collection[]
) {
if (this.allElectronAPIsExist) {
let collectionName: string;
/* collection being one suggest one of two things
1. Either the user has upload to a single existing collection
2. Created a new single collection to upload to
may have had multiple folder, but chose to upload
to one album
hence saving the collection name when upload collection count is 1
helps the info of user choosing this options
and on next upload we can directly start uploading to this collection
*/
if (collections.length === 1) {
collectionName = collections[0].name;
}
const filePaths = files.map(
(file) => (file.file as ElectronFile).path
);
this.ElectronAPIs.setToUploadFiles(filePaths);
this.ElectronAPIs.setToUploadCollection(collectionName);
}
}
updatePendingUploads(files: FileWithCollection[]) {
if (this.allElectronAPIsExist) {
const filePaths = [];
for (const fileWithCollection of files) {
if (fileWithCollection.isLivePhoto) {
filePaths.push(
(
fileWithCollection.livePhotoAssets
.image as ElectronFile
).path,
(
fileWithCollection.livePhotoAssets
.video as ElectronFile
).path
);
} else {
filePaths.push(
(fileWithCollection.file as ElectronFile).path
);
}
}
this.ElectronAPIs.setToUploadFiles(filePaths);
}
}
}
export default new ImportService();

View file

@ -1,3 +1,5 @@
import { ElectronFile } from 'types/upload';
export async function getUint8ArrayView(
reader: FileReader,
file: Blob
@ -41,6 +43,17 @@ export function getFileStream(
};
}
export async function getElectronFileStream(
file: ElectronFile,
chunkSize: number
) {
const chunkCount = Math.ceil(file.size / chunkSize);
return {
stream: await file.stream(),
chunkCount,
};
}
async function* fileChunkReaderMaker(
reader: FileReader,
file: File,

View file

@ -1,11 +1,11 @@
import { FILE_TYPE } from 'constants/file';
import { ElectronFile, FileTypeInfo } from 'types/upload';
import { FORMAT_MISSED_BY_FILE_TYPE_LIB } from 'constants/upload';
import { FileTypeInfo } from 'types/upload';
import { CustomError } from 'utils/error';
import { getFileExtension } from 'utils/file';
import { logError } from 'utils/sentry';
import { getUint8ArrayView } from './readerService';
import FileType from 'file-type/browser';
import FileType, { FileTypeResult } from 'file-type';
const TYPE_VIDEO = 'video';
const TYPE_IMAGE = 'image';
@ -13,12 +13,20 @@ const CHUNK_SIZE_FOR_TYPE_DETECTION = 4100;
export async function getFileType(
reader: FileReader,
receivedFile: File
receivedFile: File | ElectronFile
): Promise<FileTypeInfo> {
try {
let fileType: FILE_TYPE;
const typeResult = await extractFileType(reader, receivedFile);
const mimTypeParts = typeResult.mime?.split('/');
let typeResult: FileTypeResult;
if (receivedFile instanceof File) {
typeResult = await extractFileType(reader, receivedFile);
} else {
typeResult = await extractElectronFileType(receivedFile);
}
const mimTypeParts: string[] = typeResult.mime?.split('/');
if (mimTypeParts?.length !== 2) {
throw Error(CustomError.TYPE_DETECTION_FAILED);
}
@ -51,7 +59,7 @@ export async function getFileType(
return {
fileType: FILE_TYPE.OTHERS,
exactType: fileFormat,
mimeType: receivedFile.type,
mimeType: receivedFile instanceof File ? receivedFile.type : null,
};
}
}
@ -61,6 +69,14 @@ async function extractFileType(reader: FileReader, file: File) {
return getFileTypeFromBlob(reader, fileChunkBlob);
}
async function extractElectronFileType(file: ElectronFile) {
const stream = await file.stream();
const reader = stream.getReader();
const { value } = await reader.read();
const fileTypeResult = await FileType.fromBuffer(value);
return fileTypeResult;
}
async function getFileTypeFromBlob(reader: FileReader, fileBlob: Blob) {
try {
const initialFiledata = await getUint8ArrayView(reader, fileBlob);

View file

@ -9,29 +9,34 @@ import {
FileWithMetadata,
ParsedMetadataJSONMap,
DataStream,
ElectronFile,
} from 'types/upload';
import { splitFilenameAndExtension } from 'utils/file';
import { logError } from 'utils/sentry';
import { getFileNameSize, logUploadInfo } from 'utils/upload';
import { encryptFiledata } from './encryptionService';
import { extractMetadata, getMetadataJSONMapKey } from './metadataService';
import { getFileStream, getUint8ArrayView } from '../readerService';
import {
getFileStream,
getElectronFileStream,
getUint8ArrayView,
} from '../readerService';
import { generateThumbnail } from './thumbnailService';
const EDITED_FILE_SUFFIX = '-edited';
export function getFileSize(file: File) {
export function getFileSize(file: File | ElectronFile) {
return file.size;
}
export function getFilename(file: File) {
export function getFilename(file: File | ElectronFile) {
return file.name;
}
export async function readFile(
reader: FileReader,
fileTypeInfo: FileTypeInfo,
rawFile: File
rawFile: File | ElectronFile
): Promise<FileInMemory> {
const { thumbnail, hasStaticThumbnail } = await generateThumbnail(
reader,
@ -40,7 +45,16 @@ export async function readFile(
);
logUploadInfo(`reading file datal${getFileNameSize(rawFile)} `);
let filedata: Uint8Array | DataStream;
if (rawFile.size > MULTIPART_PART_SIZE) {
if (!(rawFile instanceof File)) {
if (rawFile.size > MULTIPART_PART_SIZE) {
filedata = await getElectronFileStream(
rawFile,
FILE_READER_CHUNK_SIZE
);
} else {
filedata = await rawFile.arrayBuffer();
}
} else if (rawFile.size > MULTIPART_PART_SIZE) {
filedata = getFileStream(reader, rawFile, FILE_READER_CHUNK_SIZE);
} else {
filedata = await getUint8ArrayView(reader, rawFile);
@ -57,7 +71,7 @@ export async function readFile(
export async function extractFileMetadata(
parsedMetadataJSONMap: ParsedMetadataJSONMap,
rawFile: File,
rawFile: File | ElectronFile,
collectionID: number,
fileTypeInfo: FileTypeInfo
) {
@ -121,7 +135,7 @@ export async function encryptFile(
Get the original file name for edited file to associate it to original file's metadataJSON file
as edited file doesn't have their own metadata file
*/
function getFileOriginalName(file: File) {
function getFileOriginalName(file: File | ElectronFile) {
let originalName: string = null;
const [nameWithoutExtension, extension] = splitFilenameAndExtension(
file.name

View file

@ -2,6 +2,7 @@ import { FILE_TYPE } from 'constants/file';
import { LIVE_PHOTO_ASSET_SIZE_LIMIT } from 'constants/upload';
import { encodeMotionPhoto } from 'services/motionPhotoService';
import {
ElectronFile,
FileTypeInfo,
FileWithCollection,
LivePhotoAssets,
@ -23,7 +24,7 @@ interface LivePhotoIdentifier {
}
interface Asset {
file: File;
file: File | ElectronFile;
metadata: Metadata;
fileTypeInfo: FileTypeInfo;
}
@ -78,9 +79,15 @@ export async function readLivePhoto(
}
);
const image = await getUint8ArrayView(reader, livePhotoAssets.image);
const image =
livePhotoAssets.image instanceof File
? await getUint8ArrayView(reader, livePhotoAssets.image)
: await livePhotoAssets.image.arrayBuffer();
const video = await getUint8ArrayView(reader, livePhotoAssets.video);
const video =
livePhotoAssets.video instanceof File
? await getUint8ArrayView(reader, livePhotoAssets.video)
: await livePhotoAssets.video.arrayBuffer();
return {
filedata: await encodeMotionPhoto({

View file

@ -7,6 +7,7 @@ import {
Location,
FileTypeInfo,
ParsedExtractedMetadata,
ElectronFile,
} from 'types/upload';
import { NULL_EXTRACTED_METADATA, NULL_LOCATION } from 'constants/upload';
import { splitFilenameAndExtension } from 'utils/file';
@ -26,11 +27,20 @@ const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = {
};
export async function extractMetadata(
receivedFile: File,
receivedFile: File | ElectronFile,
fileTypeInfo: FileTypeInfo
) {
let extractedMetadata: ParsedExtractedMetadata = NULL_EXTRACTED_METADATA;
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
if (!(receivedFile instanceof File)) {
receivedFile = new File(
[await receivedFile.blob()],
receivedFile.name,
{
lastModified: receivedFile.lastModified,
}
);
}
extractedMetadata = await getExifData(receivedFile, fileTypeInfo);
} else if (fileTypeInfo.fileType === FILE_TYPE.VIDEO) {
logUploadInfo(
@ -66,9 +76,15 @@ export const getMetadataJSONMapKey = (
export async function parseMetadataJSON(
reader: FileReader,
receivedFile: File
receivedFile: File | ElectronFile
) {
try {
if (!(receivedFile instanceof File)) {
receivedFile = new File(
[await receivedFile.blob()],
receivedFile.name
);
}
const metadataJSON: object = await new Promise((resolve, reject) => {
reader.onabort = () => reject(Error('file reading was aborted'));
reader.onerror = () => reject(Error('file reading has failed'));
@ -79,7 +95,7 @@ export async function parseMetadataJSON(
: reader.result;
resolve(JSON.parse(result));
};
reader.readAsText(receivedFile);
reader.readAsText(receivedFile as File);
});
const parsedMetadataJSON: ParsedMetadataJSON =

View file

@ -5,7 +5,7 @@ import { BLACK_THUMBNAIL_BASE64 } from '../../../public/images/black-thumbnail-b
import FFmpegService from 'services/ffmpeg/ffmpegService';
import { convertBytesToHumanReadable } from 'utils/billing';
import { isFileHEIC } from 'utils/file';
import { FileTypeInfo } from 'types/upload';
import { ElectronFile, FileTypeInfo } from 'types/upload';
import { getUint8ArrayView } from '../readerService';
import HEICConverter from 'services/heicConverter/heicConverterService';
import { getFileNameSize, logUploadInfo } from 'utils/upload';
@ -25,7 +25,7 @@ interface Dimension {
export async function generateThumbnail(
reader: FileReader,
file: File,
file: File | ElectronFile,
fileTypeInfo: FileTypeInfo
): Promise<{ thumbnail: Uint8Array; hasStaticThumbnail: boolean }> {
try {
@ -33,6 +33,9 @@ export async function generateThumbnail(
let hasStaticThumbnail = false;
let canvas = document.createElement('canvas');
let thumbnail: Uint8Array;
if (!(file instanceof File)) {
file = new File([await file.blob()], file.name);
}
try {
if (fileTypeInfo.fileType === FILE_TYPE.IMAGE) {
const isHEIC = isFileHEIC(fileTypeInfo.exactType);

View file

@ -1,5 +1,4 @@
import { getLocalFiles, setLocalFiles } from '../fileService';
import { getLocalCollections } from '../collectionService';
import { SetFiles } from 'types/gallery';
import { getDedicatedCryptoWorker } from 'utils/crypto';
import {
@ -9,7 +8,11 @@ import {
} from 'utils/file';
import { logError } from 'utils/sentry';
import { getMetadataJSONMapKey, parseMetadataJSON } from './metadataService';
import { getFileNameSize, segregateMetadataAndMediaFiles } from 'utils/upload';
import {
areFileWithCollectionsSame,
getFileNameSize,
segregateMetadataAndMediaFiles,
} from 'utils/upload';
import uploader from './uploader';
import UIService from './uiService';
import UploadService from './uploadService';
@ -36,6 +39,8 @@ import { getData, LS_KEYS, setData } from 'utils/storage/localStorage';
import { dedupe } from 'utils/export';
import { convertBytesToHumanReadable } from 'utils/billing';
import { logUploadInfo } from 'utils/upload';
import isElectron from 'is-electron';
import ImportService from 'services/importService';
const MAX_CONCURRENT_UPLOADS = 4;
const FILE_UPLOAD_COMPLETED = 100;
@ -45,6 +50,7 @@ class UploadManager {
private parsedMetadataJSONMap: ParsedMetadataJSONMap;
private metadataAndFileTypeInfoMap: MetadataAndFileTypeInfoMap;
private filesToBeUploaded: FileWithCollection[];
private remainingFiles: FileWithCollection[] = [];
private failedFiles: FileWithCollection[];
private existingFilesCollectionWise: Map<number, EnteFile[]>;
private existingFiles: EnteFile[];
@ -54,23 +60,26 @@ class UploadManager {
UIService.init(progressUpdater);
this.setFiles = setFiles;
}
private uploadCancelled: boolean;
private async init(newCollections?: Collection[]) {
private resetState() {
this.uploadCancelled = false;
this.filesToBeUploaded = [];
this.remainingFiles = [];
this.failedFiles = [];
this.parsedMetadataJSONMap = new Map<string, ParsedMetadataJSON>();
this.metadataAndFileTypeInfoMap = new Map<
number,
MetadataAndFileTypeInfo
>();
}
private async init(collections: Collection[]) {
this.resetState();
this.existingFiles = await getLocalFiles();
this.existingFilesCollectionWise = sortFilesIntoCollections(
this.existingFiles
);
const collections = await getLocalCollections();
if (newCollections) {
collections.push(...newCollections);
}
this.collections = new Map(
collections.map((collection) => [collection.id, collection])
);
@ -78,10 +87,10 @@ class UploadManager {
public async queueFilesForUpload(
fileWithCollectionToBeUploaded: FileWithCollection[],
newCreatedCollections?: Collection[]
collections: Collection[]
) {
try {
await this.init(newCreatedCollections);
await this.init(collections);
logUploadInfo(
`received ${fileWithCollectionToBeUploaded.length} files to upload`
);
@ -150,6 +159,9 @@ class UploadManager {
const reader = new FileReader();
for (const { file, collectionID } of metadataFiles) {
try {
if (this.uploadCancelled) {
break;
}
logUploadInfo(
`parsing metadata json file ${getFileNameSize(file)}`
);
@ -194,6 +206,9 @@ class UploadManager {
const reader = new FileReader();
for (const { file, localID, collectionID } of mediaFiles) {
try {
if (this.uploadCancelled) {
break;
}
const { fileTypeInfo, metadata } = await (async () => {
if (file.size >= MAX_FILE_SIZE_SUPPORTED) {
logUploadInfo(
@ -254,8 +269,16 @@ class UploadManager {
}
private async uploadMediaFiles(mediaFiles: FileWithCollection[]) {
if (this.uploadCancelled) {
return;
}
logUploadInfo(`uploadMediaFiles called`);
this.filesToBeUploaded.push(...mediaFiles);
if (isElectron()) {
this.remainingFiles.push(...mediaFiles);
}
UIService.reset(mediaFiles.length);
await UploadService.setFileCount(mediaFiles.length);
@ -285,6 +308,9 @@ class UploadManager {
private async uploadNextFileInQueue(worker: any, reader: FileReader) {
while (this.filesToBeUploaded.length > 0) {
if (this.uploadCancelled) {
return;
}
const fileWithCollection = this.filesToBeUploaded.pop();
const { collectionID } = fileWithCollection;
const existingFilesInCollection =
@ -329,6 +355,14 @@ class UploadManager {
this.failedFiles.push(fileWithCollection);
}
if (isElectron()) {
this.remainingFiles = this.remainingFiles.filter(
(file) =>
!areFileWithCollectionsSame(file, fileWithCollection)
);
ImportService.updatePendingUploads(this.remainingFiles);
}
UIService.moveFileToResultList(
fileWithCollection.localID,
fileUploadResult
@ -338,7 +372,14 @@ class UploadManager {
}
async retryFailedFiles() {
await this.queueFilesForUpload(this.failedFiles);
await this.queueFilesForUpload(this.failedFiles, [
...this.collections.values(),
]);
}
cancelRemainingUploads() {
this.remainingFiles = [];
this.uploadCancelled = true;
}
}

View file

@ -7,6 +7,7 @@ import { handleUploadError } from 'utils/error';
import {
B64EncryptionResult,
BackupedFile,
ElectronFile,
EncryptedFile,
FileTypeInfo,
FileWithCollection,
@ -75,7 +76,7 @@ class UploadService {
: getFilename(file);
}
async getFileType(reader: FileReader, file: File) {
async getFileType(reader: FileReader, file: File | ElectronFile) {
return getFileType(reader, file);
}
@ -90,7 +91,7 @@ class UploadService {
}
async extractFileMetadata(
file: File,
file: File | ElectronFile,
collectionID: number,
fileTypeInfo: FileTypeInfo
): Promise<Metadata> {

View file

@ -1,9 +1,15 @@
import { NULL_EXTRACTED_METADATA } from 'constants/upload';
import ffmpegService from 'services/ffmpeg/ffmpegService';
import { ElectronFile } from 'types/upload';
import { logError } from 'utils/sentry';
export async function getVideoMetadata(file: File) {
export async function getVideoMetadata(file: File | ElectronFile) {
let videoMetadata = NULL_EXTRACTED_METADATA;
if (!(file instanceof File)) {
file = new File([await file.blob()], file.name, {
lastModified: file.lastModified,
});
}
try {
videoMetadata = await ffmpegService.extractMetadata(file);
} catch (e) {

View file

@ -68,14 +68,33 @@ export interface ProgressUpdater {
setHasLivePhotos: React.Dispatch<React.SetStateAction<boolean>>;
}
/*
* ElectronFile is a custom interface that is used to represent
* any file on disk as a File-like object in the Electron desktop app.
*
* This was added to support the auto-resuming of failed uploads
* which needed absolute paths to the files which the
* normal File interface does not provide.
*/
export interface ElectronFile {
name: string;
path: string;
size: number;
lastModified: number;
stream: () => Promise<ReadableStream<Uint8Array>>;
blob: () => Promise<Blob>;
arrayBuffer: () => Promise<Uint8Array>;
}
export interface UploadAsset {
isLivePhoto?: boolean;
file?: File;
file?: File | ElectronFile;
livePhotoAssets?: LivePhotoAssets;
isElectron?: boolean;
}
export interface LivePhotoAssets {
image: globalThis.File;
video: globalThis.File;
image: globalThis.File | ElectronFile;
video: globalThis.File | ElectronFile;
}
export interface FileWithCollection extends UploadAsset {

View file

@ -108,6 +108,7 @@ const englishConstants = {
3: (fileCounter) =>
`${fileCounter.finished} / ${fileCounter.total} files backed up`,
4: 'backup complete',
5: 'cancelling remaining uploads',
},
UPLOADING_FILES: 'file upload',
FILE_NOT_UPLOADED_LIST: 'the following files were not uploaded',
@ -691,6 +692,10 @@ const englishConstants = {
PASSWORD_LOCK: 'password lock',
LOCK: 'lock',
DOWNLOAD_UPLOAD_LOGS: 'debug logs',
CHOOSE_UPLOAD_TYPE: 'Upload',
UPLOAD_FILES: 'File Upload',
UPLOAD_DIRS: 'Folder Upload',
CANCEL_UPLOADS: 'cancel uploads',
DEDUPLICATE_FILES: 'deduplicate files',
NO_DUPLICATES_FOUND: "you've no duplicate files that can be cleared",
CLUB_BY_CAPTURE_TIME: 'club by capture time',
@ -703,6 +708,10 @@ const englishConstants = {
you believe are duplicates{' '}
</>
),
STOP_ALL_UPLOADS_MESSAGE:
'are you sure that you want to stop all the uploads in progress?',
STOP_UPLOADS_HEADER: 'stop uploads?',
YES_STOP_UPLOADS: 'yes, stop uploads',
};
export default englishConstants;

View file

@ -1,8 +1,9 @@
import { FileWithCollection, Metadata } from 'types/upload';
import { ElectronFile, FileWithCollection, Metadata } from 'types/upload';
import { EnteFile } from 'types/file';
import { convertBytesToHumanReadable } from 'utils/billing';
import { formatDateTime } from 'utils/file';
import { getLogs, saveLogLine } from 'utils/storage';
import { A_SEC_IN_MICROSECONDS } from 'constants/upload';
const TYPE_JSON = 'json';
const DEDUPE_COLLECTION = new Set(['icloud library', 'icloudlibrary']);
@ -28,10 +29,18 @@ export function areFilesSame(
existingFile: Metadata,
newFile: Metadata
): boolean {
/*
* The maximum difference in the creation/modification times of two similar files is set to 1 second.
* This is because while uploading files in the web - browsers and users could have set reduced
* precision of file times to prevent timing attacks and fingerprinting.
* Context: https://developer.mozilla.org/en-US/docs/Web/API/File/lastModified#reduced_time_precision
*/
if (
existingFile.fileType === newFile.fileType &&
existingFile.creationTime === newFile.creationTime &&
existingFile.modificationTime === newFile.modificationTime &&
Math.abs(existingFile.creationTime - newFile.creationTime) <
A_SEC_IN_MICROSECONDS &&
Math.abs(existingFile.modificationTime - newFile.modificationTime) <
A_SEC_IN_MICROSECONDS &&
existingFile.title === newFile.title
) {
return true;
@ -74,6 +83,13 @@ export function getUploadLogs() {
.map((log) => `[${formatDateTime(log.timestamp)}] ${log.logLine}`);
}
export function getFileNameSize(file: File) {
export function getFileNameSize(file: File | ElectronFile) {
return `${file.name}_${convertBytesToHumanReadable(file.size)}`;
}
export function areFileWithCollectionsSame(
firstFile: FileWithCollection,
secondFile: FileWithCollection
): boolean {
return firstFile.localID === secondFile.localID;
}

View file

@ -4623,6 +4623,11 @@ react-fast-compare@^3.0.1:
resolved "https://registry.npmjs.org/react-fast-compare/-/react-fast-compare-3.2.0.tgz"
integrity sha512-rtGImPZ0YyLrscKI9xTpV8psd6I8VAtjKCzQDlzyDvqJA8XOW78TXYQwNRNd8g8JZnDu8q9Fu/1v4HPAVwVdHA==
react-icons@^4.3.1:
version "4.3.1"
resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.3.1.tgz#2fa92aebbbc71f43d2db2ed1aed07361124e91ca"
integrity sha512-cB10MXLTs3gVuXimblAdI71jrJx8njrJZmNMEMC+sQu5B/BIOmlsAjskdqpn81y8UBVEGuHODd7/ci5DvoSzTQ==
react-input-autosize@^3.0.0:
version "3.0.0"
resolved "https://registry.npmjs.org/react-input-autosize/-/react-input-autosize-3.0.0.tgz"