diff --git a/mobile/lib/db/upload_locks_db.dart b/mobile/lib/db/upload_locks_db.dart index f679911b4..def0df77a 100644 --- a/mobile/lib/db/upload_locks_db.dart +++ b/mobile/lib/db/upload_locks_db.dart @@ -4,12 +4,14 @@ import 'dart:io'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; import "package:photos/core/constants.dart"; +import "package:photos/models/encryption_result.dart"; import "package:photos/module/upload/model/multipart.dart"; +import "package:photos/utils/crypto_util.dart"; import 'package:sqflite/sqflite.dart'; +import "package:sqflite_migration/sqflite_migration.dart"; class UploadLocksDB { static const _databaseName = "ente.upload_locks.db"; - static const _databaseVersion = 1; static const _uploadLocksTable = ( table: "upload_locks", @@ -26,6 +28,7 @@ class UploadLocksDB { columnEncryptedFilePath: "encrypted_file_path", columnEncryptedFileSize: "encrypted_file_size", columnFileKey: "file_key", + columnFileNonce: "file_nonce", columnObjectKey: "object_key", columnCompleteUrl: "complete_url", columnStatus: "status", @@ -41,6 +44,19 @@ class UploadLocksDB { columnPartStatus: "part_status", ); + static final initializationScript = [ + ..._createUploadLocksTable(), + ]; + + static final migrationScripts = [ + ..._createTrackUploadsTable(), + ]; + + final dbConfig = MigrationConfig( + initializationScript: initializationScript, + migrationScripts: migrationScripts, + ); + UploadLocksDB._privateConstructor(); static final UploadLocksDB instance = UploadLocksDB._privateConstructor(); @@ -55,18 +71,11 @@ class UploadLocksDB { await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); - return await openDatabase( - path, - version: _databaseVersion, - onCreate: _onCreate, - onOpen: (db) async { - await _createTrackUploadsTable(db); - }, - ); + return await openDatabaseWithMigration(path, dbConfig); } - Future _onCreate(Database db, int version) async { - await db.execute( + static List _createUploadLocksTable() { + return [ ''' CREATE TABLE ${_uploadLocksTable.table} ( ${_uploadLocksTable.columnID} TEXT PRIMARY KEY NOT NULL, @@ -74,23 +83,11 @@ class UploadLocksDB { ${_uploadLocksTable.columnTime} TEXT NOT NULL ) ''', - ); - await _createTrackUploadsTable(db); + ]; } - Future _createTrackUploadsTable(Database db) async { - if ((await db.query( - 'sqlite_master', - where: 'name = ?', - whereArgs: [ - _trackUploadTable.table, - ], - )) - .isNotEmpty) { - return; - } - - await db.execute( + static List _createTrackUploadsTable() { + return [ ''' CREATE TABLE ${_trackUploadTable.table} ( ${_trackUploadTable.columnID} INTEGER PRIMARY KEY, @@ -99,14 +96,13 @@ class UploadLocksDB { ${_trackUploadTable.columnEncryptedFilePath} TEXT NOT NULL, ${_trackUploadTable.columnEncryptedFileSize} INTEGER NOT NULL, ${_trackUploadTable.columnFileKey} TEXT NOT NULL, + ${_trackUploadTable.columnFileNonce} TEXT NOT NULL, ${_trackUploadTable.columnObjectKey} TEXT NOT NULL, ${_trackUploadTable.columnCompleteUrl} TEXT NOT NULL, ${_trackUploadTable.columnStatus} TEXT DEFAULT '${MultipartStatus.pending.name}' NOT NULL, ${_trackUploadTable.columnPartSize} INTEGER NOT NULL ) ''', - ); - await db.execute( ''' CREATE TABLE ${_partsTable.table} ( ${_partsTable.columnObjectKey} TEXT NOT NULL REFERENCES ${_trackUploadTable.table}(${_trackUploadTable.columnObjectKey}) ON DELETE CASCADE, @@ -117,7 +113,7 @@ class UploadLocksDB { PRIMARY KEY (${_partsTable.columnObjectKey}, ${_partsTable.columnPartNumber}) ) ''', - ); + ]; } Future clearTable() async { @@ -193,6 +189,33 @@ class UploadLocksDB { return rows.isNotEmpty; } + Future getFileEncryptionData( + String localId, + String fileHash, + ) async { + final db = await instance.database; + + final rows = await db.query( + _trackUploadTable.table, + where: + '${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?', + whereArgs: [localId, fileHash], + ); + + if (rows.isEmpty) { + throw Exception("No cached links found for $localId and $fileHash"); + } + final row = rows.first; + + return EncryptionResult( + key: + CryptoUtil.base642bin(row[_trackUploadTable.columnFileKey] as String), + header: CryptoUtil.base642bin( + row[_trackUploadTable.columnFileNonce] as String, + ), + ); + } + Future getCachedLinks( String localId, String fileHash, @@ -255,6 +278,7 @@ class UploadLocksDB { String encryptedFilePath, int fileSize, String fileKey, + String fileNonce, ) async { final db = await UploadLocksDB.instance.database; final objectKey = urls.objectKey; @@ -269,6 +293,7 @@ class UploadLocksDB { _trackUploadTable.columnEncryptedFilePath: encryptedFilePath, _trackUploadTable.columnEncryptedFileSize: fileSize, _trackUploadTable.columnFileKey: fileKey, + _trackUploadTable.columnFileNonce: fileNonce, _trackUploadTable.columnPartSize: multipartPartSizeForUpload, }, ); @@ -315,14 +340,14 @@ class UploadLocksDB { await db.update( _trackUploadTable.table, { - _trackUploadTable.columnStatus: status, + _trackUploadTable.columnStatus: status.name, }, where: '${_trackUploadTable.columnObjectKey} = ?', whereArgs: [objectKey], ); } - Future deleteCompletedRecord( + Future deleteMultipartTrack( String localId, ) async { final db = await instance.database; diff --git a/mobile/lib/module/upload/service/multipart.dart b/mobile/lib/module/upload/service/multipart.dart index 3ea84bad4..97bf06082 100644 --- a/mobile/lib/module/upload/service/multipart.dart +++ b/mobile/lib/module/upload/service/multipart.dart @@ -5,6 +5,7 @@ import "package:dio/dio.dart"; import "package:logging/logging.dart"; import "package:photos/core/constants.dart"; import "package:photos/db/upload_locks_db.dart"; +import "package:photos/models/encryption_result.dart"; import "package:photos/module/upload/model/multipart.dart"; import "package:photos/module/upload/model/xml.dart"; import "package:photos/services/feature_flag_service.dart"; @@ -24,6 +25,13 @@ class MultiPartUploader { this._featureFlagService, ); + Future getEncryptionResult( + String localId, + String fileHash, + ) { + return _db.getFileEncryptionData(localId, fileHash); + } + Future calculatePartCount(int fileSize) async { final partCount = (fileSize / multipartPartSizeForUpload).ceil(); return partCount; @@ -56,6 +64,7 @@ class MultiPartUploader { String encryptedFilePath, int fileSize, Uint8List fileKey, + Uint8List fileNonce, ) async { await _db.createTrackUploadsEntry( localId, @@ -64,6 +73,7 @@ class MultiPartUploader { encryptedFilePath, fileSize, CryptoUtil.bin2base64(fileKey), + CryptoUtil.bin2base64(fileNonce), ); } @@ -118,12 +128,17 @@ class MultiPartUploader { final partsLength = partsURLs.length; final etags = partInfo.partETags ?? {}; - for (int i = 0; i < partsLength; i++) { - if (i < (partUploadStatus?.length ?? 0) && - (partUploadStatus?[i] ?? false)) { - continue; - } - final partSize = partInfo.partSize ?? multipartPartSizeForUpload; + int i = 0; + final partSize = partInfo.partSize ?? multipartPartSizeForUpload; + + // Go to the first part that is not uploaded + while (i < (partUploadStatus?.length ?? 0) && + (partUploadStatus?[i] ?? false)) { + i++; + } + + // Start parts upload + while (i < partsLength) { final partURL = partsURLs[i]; final isLastPart = i == partsLength - 1; final fileSize = @@ -151,7 +166,9 @@ class MultiPartUploader { etags[i] = eTag!; await _db.updatePartStatus(partInfo.urls.objectKey, i, eTag); + i++; } + await _db.updateTrackUploadStatus( partInfo.urls.objectKey, MultipartStatus.uploaded, diff --git a/mobile/lib/utils/file_uploader.dart b/mobile/lib/utils/file_uploader.dart index 5ed027456..f2f79088f 100644 --- a/mobile/lib/utils/file_uploader.dart +++ b/mobile/lib/utils/file_uploader.dart @@ -2,7 +2,7 @@ import 'dart:async'; import 'dart:collection'; import 'dart:convert'; import 'dart:io'; -import 'dart:math'; +import 'dart:math' as math; import 'package:collection/collection.dart'; import 'package:connectivity_plus/connectivity_plus.dart'; @@ -41,7 +41,6 @@ import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/file_util.dart"; import 'package:shared_preferences/shared_preferences.dart'; import 'package:tuple/tuple.dart'; -import "package:uuid/uuid.dart"; class FileUploader { static const kMaximumConcurrentUploads = 4; @@ -424,12 +423,19 @@ class FileUploader { } final tempDirectory = Configuration.instance.getTempDirectory(); - final String uniqueID = const Uuid().v4().toString(); + MediaUploadData? mediaUploadData; + mediaUploadData = await getUploadDataFromEnteFile(file); + + final String uniqueID = lockKey + + "_" + + mediaUploadData.hashData!.fileHash! + .replaceAll('+', '') + .replaceAll('/', ''); + final encryptedFilePath = '$tempDirectory$kUploadTempPrefix${uniqueID}_file.encrypted'; final encryptedThumbnailPath = '$tempDirectory$kUploadTempPrefix${uniqueID}_thumb.encrypted'; - MediaUploadData? mediaUploadData; var uploadCompleted = false; // This flag is used to decide whether to clear the iOS origin file cache // or not. @@ -443,13 +449,25 @@ class FileUploader { '${isUpdatedFile ? 're-upload' : 'upload'} of ${file.toString()}', ); - mediaUploadData = await getUploadDataFromEnteFile(file); + var multipartEntryExists = mediaUploadData.hashData?.fileHash != null && + await _uploadLocks.doesExists( + lockKey, + mediaUploadData.hashData!.fileHash!, + ); Uint8List? key; + EncryptionResult? multipartEncryptionResult; if (isUpdatedFile) { key = getFileKey(file); } else { - key = null; + multipartEncryptionResult = multipartEntryExists + ? await _multiPartUploader.getEncryptionResult( + lockKey, + mediaUploadData.hashData!.fileHash!, + ) + : null; + key = multipartEncryptionResult?.key; + // check if the file is already uploaded and can be mapped to existing // uploaded file. If map is found, it also returns the corresponding // mapped or update file entry. @@ -468,16 +486,30 @@ class FileUploader { } } - if (File(encryptedFilePath).existsSync()) { + final encryptedFileExists = File(encryptedFilePath).existsSync(); + + // If the multipart entry exists but the encrypted file doesn't, it means + // that we'll have to reupload as the nonce is lost + if (multipartEntryExists) { + if (!encryptedFileExists) { + await _uploadLocks.deleteMultipartTrack(lockKey); + multipartEntryExists = false; + multipartEncryptionResult = null; + } + } else if (encryptedFileExists) { + // otherwise just delete the file for singlepart upload await File(encryptedFilePath).delete(); } await _checkIfWithinStorageLimit(mediaUploadData.sourceFile!); final encryptedFile = File(encryptedFilePath); - final EncryptionResult fileAttributes = await CryptoUtil.encryptFile( - mediaUploadData.sourceFile!.path, - encryptedFilePath, - key: key, - ); + + final EncryptionResult fileAttributes = multipartEncryptionResult ?? + await CryptoUtil.encryptFile( + mediaUploadData.sourceFile!.path, + encryptedFilePath, + key: key, + ); + late final Uint8List? thumbnailData; if (mediaUploadData.thumbnail == null && file.fileType == FileType.video) { @@ -516,11 +548,7 @@ class FileUploader { final fileUploadURL = await _getUploadURL(); fileObjectKey = await _putFile(fileUploadURL, encryptedFile); } else { - if (mediaUploadData.hashData?.fileHash != null && - await _uploadLocks.doesExists( - lockKey, - mediaUploadData.hashData!.fileHash!, - )) { + if (multipartEntryExists) { fileObjectKey = await _multiPartUploader.putExistingMultipartFile( encryptedFile, lockKey, @@ -536,6 +564,7 @@ class FileUploader { encryptedFilePath, await encryptedFile.length(), fileAttributes.key!, + fileAttributes.header!, ); fileObjectKey = await _multiPartUploader.putMultipartFile( fileUploadURLs, @@ -546,7 +575,7 @@ class FileUploader { final metadata = await file.getMetadataForUpload(mediaUploadData); final encryptedMetadataResult = await CryptoUtil.encryptChaCha( - utf8.encode(jsonEncode(metadata)) as Uint8List, + utf8.encode(jsonEncode(metadata)), fileAttributes.key!, ); final fileDecryptionHeader = @@ -628,7 +657,7 @@ class FileUploader { } await FilesDB.instance.update(remoteFile); } - await UploadLocksDB.instance.deleteCompletedRecord(lockKey); + await UploadLocksDB.instance.deleteMultipartTrack(lockKey); if (!_isBackground) { Bus.instance.fire( @@ -1051,7 +1080,7 @@ class FileUploader { if (_uploadURLs.isEmpty) { // the queue is empty, fetch at least for one file to handle force uploads // that are not in the queue. This is to also avoid - await fetchUploadURLs(max(_queue.length, 1)); + await fetchUploadURLs(math.max(_queue.length, 1)); } try { return _uploadURLs.removeFirst(); @@ -1073,7 +1102,7 @@ class FileUploader { final response = await _enteDio.get( "/files/upload-urls", queryParameters: { - "count": min(42, fileCount * 2), // m4gic number + "count": math.min(42, fileCount * 2), // m4gic number }, ); final urls = (response.data["urls"] as List)