diff --git a/mobile/lib/core/constants.dart b/mobile/lib/core/constants.dart index 6f8f19115..1cd9da1dd 100644 --- a/mobile/lib/core/constants.dart +++ b/mobile/lib/core/constants.dart @@ -45,6 +45,9 @@ class FFDefault { static const bool enablePasskey = false; } +// this is the chunk size of the un-encrypted file which is read and encrypted before uploading it as a single part. +const multipartPartSize = 20 * 1024 * 1024; + const kDefaultProductionEndpoint = 'https://api.ente.io'; const int intMaxValue = 9223372036854775807; @@ -71,11 +74,11 @@ const kSearchSectionLimit = 9; const iOSGroupID = "group.io.ente.frame.SlideshowWidget"; -const blackThumbnailBase64 = '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEB' + - 'AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQ' + - 'EBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARC' + - 'ACWASwDAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF' + - 'BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk' + +const blackThumbnailBase64 = '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEB' + 'AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQ' + 'EBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARC' + 'ACWASwDAREAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUF' + 'BAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk' + '6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztL' + 'W2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAA' + 'AAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVY' + diff --git a/mobile/lib/db/device_files_db.dart b/mobile/lib/db/device_files_db.dart index 5009dc0e0..25c88daca 100644 --- a/mobile/lib/db/device_files_db.dart +++ b/mobile/lib/db/device_files_db.dart @@ -340,7 +340,7 @@ extension DeviceFiles on FilesDB { int ownerID, ) async { final db = await database; - const String rawQuery = ''' + const String rawQuery = ''' SELECT ${FilesDB.columnLocalID}, ${FilesDB.columnUploadedFileID}, ${FilesDB.columnFileSize} FROM ${FilesDB.filesTable} diff --git a/mobile/lib/utils/file_uploader.dart b/mobile/lib/utils/file_uploader.dart index 7c898f985..a545605e9 100644 --- a/mobile/lib/utils/file_uploader.dart +++ b/mobile/lib/utils/file_uploader.dart @@ -29,6 +29,7 @@ import "package:photos/models/metadata/file_magic.dart"; import 'package:photos/models/upload_url.dart'; import "package:photos/models/user_details.dart"; import 'package:photos/services/collections_service.dart'; +import "package:photos/services/feature_flag_service.dart"; import "package:photos/services/file_magic_service.dart"; import 'package:photos/services/local_sync_service.dart'; import 'package:photos/services/sync_service.dart'; @@ -37,6 +38,7 @@ import 'package:photos/utils/crypto_util.dart'; import 'package:photos/utils/file_download_util.dart'; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/file_util.dart"; +import "package:photos/utils/multipart_upload_util.dart"; import 'package:shared_preferences/shared_preferences.dart'; import 'package:tuple/tuple.dart'; import "package:uuid/uuid.dart"; @@ -492,8 +494,23 @@ class FileUploader { final String thumbnailObjectKey = await _putFile(thumbnailUploadURL, encryptedThumbnailFile); - final fileUploadURL = await _getUploadURL(); - final String fileObjectKey = await _putFile(fileUploadURL, encryptedFile); + // Calculate the number of parts for the file. Multiple part upload + // is only enabled for internal users and debug builds till it's battle tested. + final count = FeatureFlagService.instance.isInternalUserOrDebugBuild() + ? await calculatePartCount( + await encryptedFile.length(), + ) + : 1; + + late String fileObjectKey; + + if (count <= 1) { + final fileUploadURL = await _getUploadURL(); + fileObjectKey = await _putFile(fileUploadURL, encryptedFile); + } else { + final fileUploadURLs = await getMultipartUploadURLs(count); + fileObjectKey = await putMultipartFile(fileUploadURLs, encryptedFile); + } final metadata = await file.getMetadataForUpload(mediaUploadData); final encryptedMetadataResult = await CryptoUtil.encryptChaCha( diff --git a/mobile/lib/utils/multipart_upload_util.dart b/mobile/lib/utils/multipart_upload_util.dart new file mode 100644 index 000000000..6e0eda8ca --- /dev/null +++ b/mobile/lib/utils/multipart_upload_util.dart @@ -0,0 +1,157 @@ +// ignore_for_file: implementation_imports + +import "dart:io"; + +import "package:dio/dio.dart"; +import "package:logging/logging.dart"; +import "package:photos/core/constants.dart"; +import "package:photos/core/network/network.dart"; +import "package:photos/services/feature_flag_service.dart"; +import "package:photos/utils/xml_parser_util.dart"; + +final _enteDio = NetworkClient.instance.enteDio; +final _dio = NetworkClient.instance.getDio(); + +class PartETag extends XmlParsableObject { + final int partNumber; + final String eTag; + + PartETag(this.partNumber, this.eTag); + + @override + String get elementName => "Part"; + + @override + Map toMap() { + return { + "PartNumber": partNumber, + "ETag": eTag, + }; + } +} + +class MultipartUploadURLs { + final String objectKey; + final List partsURLs; + final String completeURL; + + MultipartUploadURLs({ + required this.objectKey, + required this.partsURLs, + required this.completeURL, + }); + + factory MultipartUploadURLs.fromMap(Map map) { + return MultipartUploadURLs( + objectKey: map["urls"]["objectKey"], + partsURLs: (map["urls"]["partURLs"] as List).cast(), + completeURL: map["urls"]["completeURL"], + ); + } +} + +Future calculatePartCount(int fileSize) async { + final partCount = (fileSize / multipartPartSize).ceil(); + return partCount; +} + +Future getMultipartUploadURLs(int count) async { + try { + assert( + FeatureFlagService.instance.isInternalUserOrDebugBuild(), + "Multipart upload should not be enabled for external users.", + ); + final response = await _enteDio.get( + "/files/multipart-upload-urls", + queryParameters: { + "count": count, + }, + ); + + return MultipartUploadURLs.fromMap(response.data); + } on Exception catch (e) { + Logger("MultipartUploadURL").severe(e); + rethrow; + } +} + +Future putMultipartFile( + MultipartUploadURLs urls, + File encryptedFile, +) async { + // upload individual parts and get their etags + final etags = await uploadParts(urls.partsURLs, encryptedFile); + + // complete the multipart upload + await completeMultipartUpload(etags, urls.completeURL); + + return urls.objectKey; +} + +Future> uploadParts( + List partsURLs, + File encryptedFile, +) async { + final partsLength = partsURLs.length; + final etags = {}; + + for (int i = 0; i < partsLength; i++) { + final partURL = partsURLs[i]; + final isLastPart = i == partsLength - 1; + final fileSize = isLastPart + ? encryptedFile.lengthSync() % multipartPartSize + : multipartPartSize; + + final response = await _dio.put( + partURL, + data: encryptedFile.openRead( + i * multipartPartSize, + isLastPart ? null : multipartPartSize, + ), + options: Options( + headers: { + Headers.contentLengthHeader: fileSize, + }, + ), + ); + + final eTag = response.headers.value("etag"); + + if (eTag?.isEmpty ?? true) { + throw Exception('ETAG_MISSING'); + } + + etags[i] = eTag!; + } + + return etags; +} + +Future completeMultipartUpload( + Map partEtags, + String completeURL, +) async { + final body = convertJs2Xml({ + 'CompleteMultipartUpload': partEtags.entries + .map( + (e) => PartETag( + e.key + 1, + e.value, + ), + ) + .toList(), + }).replaceAll('"', '').replaceAll('"', ''); + + try { + await _dio.post( + completeURL, + data: body, + options: Options( + contentType: "text/xml", + ), + ); + } catch (e) { + Logger("MultipartUpload").severe(e); + rethrow; + } +} diff --git a/mobile/lib/utils/xml_parser_util.dart b/mobile/lib/utils/xml_parser_util.dart new file mode 100644 index 000000000..9490fc40c --- /dev/null +++ b/mobile/lib/utils/xml_parser_util.dart @@ -0,0 +1,41 @@ +// ignore_for_file: implementation_imports + +import "package:xml/xml.dart"; + +// used for classes that can be converted to xml +abstract class XmlParsableObject { + Map toMap(); + String get elementName; +} + +// for converting the response to xml +String convertJs2Xml(Map json) { + final builder = XmlBuilder(); + buildXml(builder, json); + return builder.buildDocument().toXmlString( + pretty: true, + indent: ' ', + ); +} + +// for building the xml node tree recursively +void buildXml(XmlBuilder builder, dynamic node) { + if (node is Map) { + node.forEach((key, value) { + builder.element(key, nest: () => buildXml(builder, value)); + }); + } else if (node is List) { + for (var item in node) { + buildXml(builder, item); + } + } else if (node is XmlParsableObject) { + builder.element( + node.elementName, + nest: () { + buildXml(builder, node.toMap()); + }, + ); + } else { + builder.text(node.toString()); + } +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index f5a73696b..ccb0775d7 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -745,10 +745,10 @@ packages: dependency: "direct main" description: name: flutter_local_notifications - sha256: f9a05409385b77b06c18f200a41c7c2711ebf7415669350bb0f8474c07bd40d1 + sha256: a701df4866f9a38bb8e4450a54c143bbeeb0ce2381e7df5a36e1006f3b43bb28 url: "https://pub.dev" source: hosted - version: "17.0.0" + version: "17.0.1" flutter_local_notifications_linux: dependency: transitive description: @@ -2584,7 +2584,7 @@ packages: source: hosted version: "1.0.4" xml: - dependency: transitive + dependency: "direct main" description: name: xml sha256: b015a8ad1c488f66851d762d3090a21c600e479dc75e68328c52774040cf9226 diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index d3fbb7b67..3be873cc5 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -172,6 +172,7 @@ dependencies: wallpaper_manager_flutter: ^0.0.2 wechat_assets_picker: ^8.6.3 widgets_to_image: ^0.0.2 + xml: ^6.3.0 dependency_overrides: connectivity_plus: ^4.0.0