fix: don't re-encrypt file, add nonce field, upload parts logic

mobile/lib/db/upload_locks_db.dart

@@ -4,12 +4,14 @@ import 'dart:io';
 import 'package:path/path.dart';
 import 'package:path_provider/path_provider.dart';
 import "package:photos/core/constants.dart";
+import "package:photos/models/encryption_result.dart";
 import "package:photos/module/upload/model/multipart.dart";
+import "package:photos/utils/crypto_util.dart";
 import 'package:sqflite/sqflite.dart';
+import "package:sqflite_migration/sqflite_migration.dart";
 
 class UploadLocksDB {
   static const _databaseName = "ente.upload_locks.db";
-  static const _databaseVersion = 1;
 
   static const _uploadLocksTable = (
     table: "upload_locks",
@@ -26,6 +28,7 @@ class UploadLocksDB {
     columnEncryptedFilePath: "encrypted_file_path",
     columnEncryptedFileSize: "encrypted_file_size",
     columnFileKey: "file_key",
+    columnFileNonce: "file_nonce",
     columnObjectKey: "object_key",
     columnCompleteUrl: "complete_url",
     columnStatus: "status",
@@ -41,6 +44,19 @@ class UploadLocksDB {
     columnPartStatus: "part_status",
   );
 
+  static final initializationScript = [
+    ..._createUploadLocksTable(),
+  ];
+
+  static final migrationScripts = [
+    ..._createTrackUploadsTable(),
+  ];
+
+  final dbConfig = MigrationConfig(
+    initializationScript: initializationScript,
+    migrationScripts: migrationScripts,
+  );
+
   UploadLocksDB._privateConstructor();
   static final UploadLocksDB instance = UploadLocksDB._privateConstructor();
 
@@ -55,18 +71,11 @@ class UploadLocksDB {
         await getApplicationDocumentsDirectory();
     final String path = join(documentsDirectory.path, _databaseName);
 
-    return await openDatabase(
-      path,
-      version: _databaseVersion,
-      onCreate: _onCreate,
-      onOpen: (db) async {
-        await _createTrackUploadsTable(db);
-      },
-    );
+    return await openDatabaseWithMigration(path, dbConfig);
   }
 
-  Future _onCreate(Database db, int version) async {
-    await db.execute(
+  static List<String> _createUploadLocksTable() {
+    return [
       '''
                 CREATE TABLE ${_uploadLocksTable.table} (
                   ${_uploadLocksTable.columnID} TEXT PRIMARY KEY NOT NULL,
@@ -74,23 +83,11 @@ class UploadLocksDB {
                  ${_uploadLocksTable.columnTime} TEXT NOT NULL
                 )
                 ''',
-    );
-    await _createTrackUploadsTable(db);
+    ];
   }
 
-  Future _createTrackUploadsTable(Database db) async {
-    if ((await db.query(
-      'sqlite_master',
-      where: 'name = ?',
-      whereArgs: [
-        _trackUploadTable.table,
-      ],
-    ))
-        .isNotEmpty) {
-      return;
-    }
-
-    await db.execute(
+  static List<String> _createTrackUploadsTable() {
+    return [
       '''
                 CREATE TABLE ${_trackUploadTable.table} (
                   ${_trackUploadTable.columnID} INTEGER PRIMARY KEY,
@@ -99,14 +96,13 @@ class UploadLocksDB {
                   ${_trackUploadTable.columnEncryptedFilePath} TEXT NOT NULL,
                   ${_trackUploadTable.columnEncryptedFileSize} INTEGER NOT NULL,
                   ${_trackUploadTable.columnFileKey} TEXT NOT NULL,
+                  ${_trackUploadTable.columnFileNonce} TEXT NOT NULL,
                   ${_trackUploadTable.columnObjectKey} TEXT NOT NULL,
                   ${_trackUploadTable.columnCompleteUrl} TEXT NOT NULL,
                   ${_trackUploadTable.columnStatus} TEXT DEFAULT '${MultipartStatus.pending.name}' NOT NULL,
                   ${_trackUploadTable.columnPartSize} INTEGER NOT NULL
                 )
                 ''',
-    );
-    await db.execute(
       '''
                 CREATE TABLE ${_partsTable.table} (
                   ${_partsTable.columnObjectKey} TEXT NOT NULL REFERENCES ${_trackUploadTable.table}(${_trackUploadTable.columnObjectKey}) ON DELETE CASCADE,
@@ -117,7 +113,7 @@ class UploadLocksDB {
                   PRIMARY KEY (${_partsTable.columnObjectKey}, ${_partsTable.columnPartNumber})
                 )
                 ''',
-    );
+    ];
   }
 
   Future<void> clearTable() async {
@@ -193,6 +189,33 @@ class UploadLocksDB {
     return rows.isNotEmpty;
   }
 
+  Future<EncryptionResult> getFileEncryptionData(
+    String localId,
+    String fileHash,
+  ) async {
+    final db = await instance.database;
+
+    final rows = await db.query(
+      _trackUploadTable.table,
+      where:
+          '${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?',
+      whereArgs: [localId, fileHash],
+    );
+
+    if (rows.isEmpty) {
+      throw Exception("No cached links found for $localId and $fileHash");
+    }
+    final row = rows.first;
+
+    return EncryptionResult(
+      key:
+          CryptoUtil.base642bin(row[_trackUploadTable.columnFileKey] as String),
+      header: CryptoUtil.base642bin(
+        row[_trackUploadTable.columnFileNonce] as String,
+      ),
+    );
+  }
+
   Future<MultipartInfo> getCachedLinks(
     String localId,
     String fileHash,
@@ -255,6 +278,7 @@ class UploadLocksDB {
     String encryptedFilePath,
     int fileSize,
     String fileKey,
+    String fileNonce,
   ) async {
     final db = await UploadLocksDB.instance.database;
     final objectKey = urls.objectKey;
@@ -269,6 +293,7 @@ class UploadLocksDB {
         _trackUploadTable.columnEncryptedFilePath: encryptedFilePath,
         _trackUploadTable.columnEncryptedFileSize: fileSize,
         _trackUploadTable.columnFileKey: fileKey,
+        _trackUploadTable.columnFileNonce: fileNonce,
         _trackUploadTable.columnPartSize: multipartPartSizeForUpload,
       },
     );
@@ -315,14 +340,14 @@ class UploadLocksDB {
     await db.update(
       _trackUploadTable.table,
       {
-        _trackUploadTable.columnStatus: status,
+        _trackUploadTable.columnStatus: status.name,
       },
       where: '${_trackUploadTable.columnObjectKey} = ?',
       whereArgs: [objectKey],
     );
   }
 
-  Future<int> deleteCompletedRecord(
+  Future<int> deleteMultipartTrack(
     String localId,
   ) async {
     final db = await instance.database;

mobile/lib/module/upload/service/multipart.dart

@@ -5,6 +5,7 @@ import "package:dio/dio.dart";
 import "package:logging/logging.dart";
 import "package:photos/core/constants.dart";
 import "package:photos/db/upload_locks_db.dart";
+import "package:photos/models/encryption_result.dart";
 import "package:photos/module/upload/model/multipart.dart";
 import "package:photos/module/upload/model/xml.dart";
 import "package:photos/services/feature_flag_service.dart";
@@ -24,6 +25,13 @@ class MultiPartUploader {
     this._featureFlagService,
   );
 
+  Future<EncryptionResult> getEncryptionResult(
+    String localId,
+    String fileHash,
+  ) {
+    return _db.getFileEncryptionData(localId, fileHash);
+  }
+
   Future<int> calculatePartCount(int fileSize) async {
     final partCount = (fileSize / multipartPartSizeForUpload).ceil();
     return partCount;
@@ -56,6 +64,7 @@ class MultiPartUploader {
     String encryptedFilePath,
     int fileSize,
     Uint8List fileKey,
+    Uint8List fileNonce,
   ) async {
     await _db.createTrackUploadsEntry(
       localId,
@@ -64,6 +73,7 @@ class MultiPartUploader {
       encryptedFilePath,
       fileSize,
       CryptoUtil.bin2base64(fileKey),
+      CryptoUtil.bin2base64(fileNonce),
     );
   }
 
@@ -118,12 +128,17 @@ class MultiPartUploader {
     final partsLength = partsURLs.length;
     final etags = partInfo.partETags ?? <int, String>{};
 
-    for (int i = 0; i < partsLength; i++) {
-      if (i < (partUploadStatus?.length ?? 0) &&
-          (partUploadStatus?[i] ?? false)) {
-        continue;
-      }
-      final partSize = partInfo.partSize ?? multipartPartSizeForUpload;
+    int i = 0;
+    final partSize = partInfo.partSize ?? multipartPartSizeForUpload;
+
+    // Go to the first part that is not uploaded
+    while (i < (partUploadStatus?.length ?? 0) &&
+        (partUploadStatus?[i] ?? false)) {
+      i++;
+    }
+
+    // Start parts upload
+    while (i < partsLength) {
       final partURL = partsURLs[i];
       final isLastPart = i == partsLength - 1;
       final fileSize =
@@ -151,7 +166,9 @@ class MultiPartUploader {
       etags[i] = eTag!;
 
       await _db.updatePartStatus(partInfo.urls.objectKey, i, eTag);
+      i++;
     }
+
     await _db.updateTrackUploadStatus(
       partInfo.urls.objectKey,
       MultipartStatus.uploaded,

mobile/lib/utils/file_uploader.dart

@@ -2,7 +2,7 @@ import 'dart:async';
 import 'dart:collection';
 import 'dart:convert';
 import 'dart:io';
-import 'dart:math';
+import 'dart:math' as math;
 
 import 'package:collection/collection.dart';
 import 'package:connectivity_plus/connectivity_plus.dart';
@@ -41,7 +41,6 @@ import 'package:photos/utils/file_uploader_util.dart';
 import "package:photos/utils/file_util.dart";
 import 'package:shared_preferences/shared_preferences.dart';
 import 'package:tuple/tuple.dart';
-import "package:uuid/uuid.dart";
 
 class FileUploader {
   static const kMaximumConcurrentUploads = 4;
@@ -424,12 +423,19 @@ class FileUploader {
     }
 
     final tempDirectory = Configuration.instance.getTempDirectory();
-    final String uniqueID = const Uuid().v4().toString();
+    MediaUploadData? mediaUploadData;
+    mediaUploadData = await getUploadDataFromEnteFile(file);
+
+    final String uniqueID = lockKey +
+        "_" +
+        mediaUploadData.hashData!.fileHash!
+            .replaceAll('+', '')
+            .replaceAll('/', '');
+
     final encryptedFilePath =
         '$tempDirectory$kUploadTempPrefix${uniqueID}_file.encrypted';
     final encryptedThumbnailPath =
         '$tempDirectory$kUploadTempPrefix${uniqueID}_thumb.encrypted';
-    MediaUploadData? mediaUploadData;
     var uploadCompleted = false;
     // This flag is used to decide whether to clear the iOS origin file cache
     // or not.
@@ -443,13 +449,25 @@ class FileUploader {
         '${isUpdatedFile ? 're-upload' : 'upload'} of ${file.toString()}',
       );
 
-      mediaUploadData = await getUploadDataFromEnteFile(file);
+      var multipartEntryExists = mediaUploadData.hashData?.fileHash != null &&
+          await _uploadLocks.doesExists(
+            lockKey,
+            mediaUploadData.hashData!.fileHash!,
+          );
 
       Uint8List? key;
+      EncryptionResult? multipartEncryptionResult;
       if (isUpdatedFile) {
         key = getFileKey(file);
       } else {
-        key = null;
+        multipartEncryptionResult = multipartEntryExists
+            ? await _multiPartUploader.getEncryptionResult(
+                lockKey,
+                mediaUploadData.hashData!.fileHash!,
+              )
+            : null;
+        key = multipartEncryptionResult?.key;
+
         // check if the file is already uploaded and can be mapped to existing
         // uploaded file. If map is found, it also returns the corresponding
         // mapped or update file entry.
@@ -468,16 +486,30 @@ class FileUploader {
         }
       }
 
-      if (File(encryptedFilePath).existsSync()) {
+      final encryptedFileExists = File(encryptedFilePath).existsSync();
+
+      // If the multipart entry exists but the encrypted file doesn't, it means
+      // that we'll have to reupload as the nonce is lost
+      if (multipartEntryExists) {
+        if (!encryptedFileExists) {
+          await _uploadLocks.deleteMultipartTrack(lockKey);
+          multipartEntryExists = false;
+          multipartEncryptionResult = null;
+        }
+      } else if (encryptedFileExists) {
+        // otherwise just delete the file for singlepart upload
         await File(encryptedFilePath).delete();
       }
       await _checkIfWithinStorageLimit(mediaUploadData.sourceFile!);
       final encryptedFile = File(encryptedFilePath);
-      final EncryptionResult fileAttributes = await CryptoUtil.encryptFile(
-        mediaUploadData.sourceFile!.path,
-        encryptedFilePath,
-        key: key,
-      );
+
+      final EncryptionResult fileAttributes = multipartEncryptionResult ??
+          await CryptoUtil.encryptFile(
+            mediaUploadData.sourceFile!.path,
+            encryptedFilePath,
+            key: key,
+          );
+
       late final Uint8List? thumbnailData;
       if (mediaUploadData.thumbnail == null &&
           file.fileType == FileType.video) {
@@ -516,11 +548,7 @@ class FileUploader {
         final fileUploadURL = await _getUploadURL();
         fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
       } else {
-        if (mediaUploadData.hashData?.fileHash != null &&
-            await _uploadLocks.doesExists(
-              lockKey,
-              mediaUploadData.hashData!.fileHash!,
-            )) {
+        if (multipartEntryExists) {
           fileObjectKey = await _multiPartUploader.putExistingMultipartFile(
             encryptedFile,
             lockKey,
@@ -536,6 +564,7 @@ class FileUploader {
             encryptedFilePath,
             await encryptedFile.length(),
             fileAttributes.key!,
+            fileAttributes.header!,
           );
           fileObjectKey = await _multiPartUploader.putMultipartFile(
             fileUploadURLs,
@@ -546,7 +575,7 @@ class FileUploader {
 
       final metadata = await file.getMetadataForUpload(mediaUploadData);
       final encryptedMetadataResult = await CryptoUtil.encryptChaCha(
-        utf8.encode(jsonEncode(metadata)) as Uint8List,
+        utf8.encode(jsonEncode(metadata)),
         fileAttributes.key!,
       );
       final fileDecryptionHeader =
@@ -628,7 +657,7 @@ class FileUploader {
         }
         await FilesDB.instance.update(remoteFile);
       }
-      await UploadLocksDB.instance.deleteCompletedRecord(lockKey);
+      await UploadLocksDB.instance.deleteMultipartTrack(lockKey);
 
       if (!_isBackground) {
         Bus.instance.fire(
@@ -1051,7 +1080,7 @@ class FileUploader {
     if (_uploadURLs.isEmpty) {
       // the queue is empty, fetch at least for one file to handle force uploads
       // that are not in the queue. This is to also avoid
-      await fetchUploadURLs(max(_queue.length, 1));
+      await fetchUploadURLs(math.max(_queue.length, 1));
     }
     try {
       return _uploadURLs.removeFirst();
@@ -1073,7 +1102,7 @@ class FileUploader {
         final response = await _enteDio.get(
           "/files/upload-urls",
           queryParameters: {
-            "count": min(42, fileCount * 2), // m4gic number
+            "count": math.min(42, fileCount * 2), // m4gic number
           },
         );
         final urls = (response.data["urls"] as List)