[FEAT] Multipart upload support (#1347)

## Description

Add multipart file upload support

mobile/lib/core/constants.dart

@@ -45,6 +45,9 @@ class FFDefault {
   static const bool enablePasskey = false;
 }
 
+// this is the chunk size of the un-encrypted file which is read and encrypted before uploading it as a single part.
+const multipartPartSize = 20 * 1024 * 1024;
+
 const kDefaultProductionEndpoint = 'https://api.ente.io';
 
 const int intMaxValue = 9223372036854775807;
@@ -71,11 +74,11 @@ const kSearchSectionLimit = 9;
 
 const iOSGroupID = "group.io.ente.frame.SlideshowWidget";
 
-const blackThumbnailBase64 = '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEB' +
-    'AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQ' +
-    'EBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARC' +
-    'ACWASwDAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF' +
-    'BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk' +
+const blackThumbnailBase64 = '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEB'
+        'AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQ'
+        'EBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARC'
+        'ACWASwDAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF'
+        'BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk' +
     '6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztL' +
     'W2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAA' +
     'AAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVY' +

mobile/lib/db/device_files_db.dart

@@ -340,7 +340,7 @@ extension DeviceFiles on FilesDB {
     int ownerID,
   ) async {
     final db = await database;
-    const String rawQuery = ''' 
+    const String rawQuery = '''
     SELECT ${FilesDB.columnLocalID}, ${FilesDB.columnUploadedFileID}, 
     ${FilesDB.columnFileSize} 
     FROM ${FilesDB.filesTable}

mobile/lib/utils/file_uploader.dart

@@ -29,6 +29,7 @@ import "package:photos/models/metadata/file_magic.dart";
 import 'package:photos/models/upload_url.dart';
 import "package:photos/models/user_details.dart";
 import 'package:photos/services/collections_service.dart';
+import "package:photos/services/feature_flag_service.dart";
 import "package:photos/services/file_magic_service.dart";
 import 'package:photos/services/local_sync_service.dart';
 import 'package:photos/services/sync_service.dart';
@@ -37,6 +38,7 @@ import 'package:photos/utils/crypto_util.dart';
 import 'package:photos/utils/file_download_util.dart';
 import 'package:photos/utils/file_uploader_util.dart';
 import "package:photos/utils/file_util.dart";
+import "package:photos/utils/multipart_upload_util.dart";
 import 'package:shared_preferences/shared_preferences.dart';
 import 'package:tuple/tuple.dart';
 import "package:uuid/uuid.dart";
@@ -492,8 +494,23 @@ class FileUploader {
       final String thumbnailObjectKey =
           await _putFile(thumbnailUploadURL, encryptedThumbnailFile);
 
-      final fileUploadURL = await _getUploadURL();
-      final String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
+      // Calculate the number of parts for the file. Multiple part upload
+      // is only enabled for internal users and debug builds till it's battle tested.
+      final count = FeatureFlagService.instance.isInternalUserOrDebugBuild()
+          ? await calculatePartCount(
+              await encryptedFile.length(),
+            )
+          : 1;
+
+      late String fileObjectKey;
+
+      if (count <= 1) {
+        final fileUploadURL = await _getUploadURL();
+        fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
+      } else {
+        final fileUploadURLs = await getMultipartUploadURLs(count);
+        fileObjectKey = await putMultipartFile(fileUploadURLs, encryptedFile);
+      }
 
       final metadata = await file.getMetadataForUpload(mediaUploadData);
       final encryptedMetadataResult = await CryptoUtil.encryptChaCha(

mobile/lib/utils/multipart_upload_util.dart

@@ -0,0 +1,157 @@
+// ignore_for_file: implementation_imports
+
+import "dart:io";
+
+import "package:dio/dio.dart";
+import "package:logging/logging.dart";
+import "package:photos/core/constants.dart";
+import "package:photos/core/network/network.dart";
+import "package:photos/services/feature_flag_service.dart";
+import "package:photos/utils/xml_parser_util.dart";
+
+final _enteDio = NetworkClient.instance.enteDio;
+final _dio = NetworkClient.instance.getDio();
+
+class PartETag extends XmlParsableObject {
+  final int partNumber;
+  final String eTag;
+
+  PartETag(this.partNumber, this.eTag);
+
+  @override
+  String get elementName => "Part";
+
+  @override
+  Map<String, dynamic> toMap() {
+    return {
+      "PartNumber": partNumber,
+      "ETag": eTag,
+    };
+  }
+}
+
+class MultipartUploadURLs {
+  final String objectKey;
+  final List<String> partsURLs;
+  final String completeURL;
+
+  MultipartUploadURLs({
+    required this.objectKey,
+    required this.partsURLs,
+    required this.completeURL,
+  });
+
+  factory MultipartUploadURLs.fromMap(Map<String, dynamic> map) {
+    return MultipartUploadURLs(
+      objectKey: map["urls"]["objectKey"],
+      partsURLs: (map["urls"]["partURLs"] as List).cast<String>(),
+      completeURL: map["urls"]["completeURL"],
+    );
+  }
+}
+
+Future<int> calculatePartCount(int fileSize) async {
+  final partCount = (fileSize / multipartPartSize).ceil();
+  return partCount;
+}
+
+Future<MultipartUploadURLs> getMultipartUploadURLs(int count) async {
+  try {
+    assert(
+      FeatureFlagService.instance.isInternalUserOrDebugBuild(),
+      "Multipart upload should not be enabled for external users.",
+    );
+    final response = await _enteDio.get(
+      "/files/multipart-upload-urls",
+      queryParameters: {
+        "count": count,
+      },
+    );
+
+    return MultipartUploadURLs.fromMap(response.data);
+  } on Exception catch (e) {
+    Logger("MultipartUploadURL").severe(e);
+    rethrow;
+  }
+}
+
+Future<String> putMultipartFile(
+  MultipartUploadURLs urls,
+  File encryptedFile,
+) async {
+  // upload individual parts and get their etags
+  final etags = await uploadParts(urls.partsURLs, encryptedFile);
+
+  // complete the multipart upload
+  await completeMultipartUpload(etags, urls.completeURL);
+
+  return urls.objectKey;
+}
+
+Future<Map<int, String>> uploadParts(
+  List<String> partsURLs,
+  File encryptedFile,
+) async {
+  final partsLength = partsURLs.length;
+  final etags = <int, String>{};
+
+  for (int i = 0; i < partsLength; i++) {
+    final partURL = partsURLs[i];
+    final isLastPart = i == partsLength - 1;
+    final fileSize = isLastPart
+        ? encryptedFile.lengthSync() % multipartPartSize
+        : multipartPartSize;
+
+    final response = await _dio.put(
+      partURL,
+      data: encryptedFile.openRead(
+        i * multipartPartSize,
+        isLastPart ? null : multipartPartSize,
+      ),
+      options: Options(
+        headers: {
+          Headers.contentLengthHeader: fileSize,
+        },
+      ),
+    );
+
+    final eTag = response.headers.value("etag");
+
+    if (eTag?.isEmpty ?? true) {
+      throw Exception('ETAG_MISSING');
+    }
+
+    etags[i] = eTag!;
+  }
+
+  return etags;
+}
+
+Future<void> completeMultipartUpload(
+  Map<int, String> partEtags,
+  String completeURL,
+) async {
+  final body = convertJs2Xml({
+    'CompleteMultipartUpload': partEtags.entries
+        .map(
+          (e) => PartETag(
+            e.key + 1,
+            e.value,
+          ),
+        )
+        .toList(),
+  }).replaceAll('"', '').replaceAll('&quot;', '');
+
+  try {
+    await _dio.post(
+      completeURL,
+      data: body,
+      options: Options(
+        contentType: "text/xml",
+      ),
+    );
+  } catch (e) {
+    Logger("MultipartUpload").severe(e);
+    rethrow;
+  }
+}

mobile/lib/utils/xml_parser_util.dart

@@ -0,0 +1,41 @@
+// ignore_for_file: implementation_imports
+
+import "package:xml/xml.dart";
+
+// used for classes that can be converted to xml
+abstract class XmlParsableObject {
+  Map<String, dynamic> toMap();
+  String get elementName;
+}
+
+// for converting the response to xml
+String convertJs2Xml(Map<String, dynamic> json) {
+  final builder = XmlBuilder();
+  buildXml(builder, json);
+  return builder.buildDocument().toXmlString(
+        pretty: true,
+        indent: '    ',
+      );
+}
+
+// for building the xml node tree recursively
+void buildXml(XmlBuilder builder, dynamic node) {
+  if (node is Map<String, dynamic>) {
+    node.forEach((key, value) {
+      builder.element(key, nest: () => buildXml(builder, value));
+    });
+  } else if (node is List<dynamic>) {
+    for (var item in node) {
+      buildXml(builder, item);
+    }
+  } else if (node is XmlParsableObject) {
+    builder.element(
+      node.elementName,
+      nest: () {
+        buildXml(builder, node.toMap());
+      },
+    );
+  } else {
+    builder.text(node.toString());
+  }
+}

mobile/pubspec.lock

@@ -745,10 +745,10 @@ packages:
     dependency: "direct main"
     description:
       name: flutter_local_notifications
-      sha256: f9a05409385b77b06c18f200a41c7c2711ebf7415669350bb0f8474c07bd40d1
+      sha256: a701df4866f9a38bb8e4450a54c143bbeeb0ce2381e7df5a36e1006f3b43bb28
       url: "https://pub.dev"
     source: hosted
-    version: "17.0.0"
+    version: "17.0.1"
   flutter_local_notifications_linux:
     dependency: transitive
     description:
@@ -2584,7 +2584,7 @@ packages:
     source: hosted
     version: "1.0.4"
   xml:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: xml
       sha256: b015a8ad1c488f66851d762d3090a21c600e479dc75e68328c52774040cf9226

mobile/pubspec.yaml

@@ -172,6 +172,7 @@ dependencies:
   wallpaper_manager_flutter: ^0.0.2
   wechat_assets_picker: ^8.6.3
   widgets_to_image: ^0.0.2
+  xml: ^6.3.0
 
 dependency_overrides:
   connectivity_plus: ^4.0.0