file_uploader.dart 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. import 'dart:async';
  2. import 'dart:collection';
  3. import 'dart:convert';
  4. import 'dart:io' as io;
  5. import 'package:dio/dio.dart';
  6. import 'package:flutter_sodium/flutter_sodium.dart';
  7. import 'package:logging/logging.dart';
  8. import 'package:photos/core/configuration.dart';
  9. import 'package:photos/core/constants.dart';
  10. import 'package:photos/db/files_db.dart';
  11. import 'package:photos/models/file.dart';
  12. import 'package:photos/models/location.dart';
  13. import 'package:photos/models/upload_url.dart';
  14. import 'package:photos/services/collections_service.dart';
  15. import 'package:photos/utils/crypto_util.dart';
  16. class FileUploader {
  17. final _logger = Logger("FileUploader");
  18. final _dio = Dio();
  19. final _queue = LinkedHashMap<int, FileUploadItem>();
  20. final _maximumConcurrentUploads = 4;
  21. int _currentlyUploading = 0;
  22. FileUploader._privateConstructor();
  23. static FileUploader instance = FileUploader._privateConstructor();
  24. Future<File> upload(File file, int collectionID) {
  25. // If the file hasn't been queued yet, queue it
  26. if (!_queue.containsKey(file.generatedID)) {
  27. final completer = Completer<File>();
  28. _queue[file.generatedID] = FileUploadItem(file, collectionID, completer);
  29. _pollQueue();
  30. return completer.future;
  31. }
  32. // If the file exists in the queue for a matching collectionID,
  33. // return the existing future
  34. final item = _queue[file.generatedID];
  35. if (item.collectionID == collectionID) {
  36. return item.completer.future;
  37. }
  38. // Else wait for the existing upload to complete,
  39. // and add it to the relevant collection
  40. return item.completer.future.then((uploadedFile) {
  41. return CollectionsService.instance
  42. .addToCollection(collectionID, [uploadedFile]).then((aVoid) {
  43. return uploadedFile;
  44. });
  45. });
  46. }
  47. Future<File> getCurrentUploadStatus(File file) {
  48. return _queue[file.generatedID]?.completer?.future;
  49. }
  50. Future<File> forceUpload(File file, int collectionID) async {
  51. // If the file hasn't been queued yet, ez.
  52. if (!_queue.containsKey(file.generatedID)) {
  53. return _encryptAndUploadFileToCollection(file, collectionID,
  54. forcedUpload: true);
  55. }
  56. var item = _queue[file.generatedID];
  57. // If the file is being uploaded right now, wait and proceed
  58. if (item.status == UploadStatus.in_progress) {
  59. return item.completer.future.then((uploadedFile) async {
  60. if (uploadedFile.collectionID == collectionID) {
  61. // Do nothing
  62. return uploadedFile;
  63. } else {
  64. return CollectionsService.instance
  65. .addToCollection(collectionID, [uploadedFile]).then((aVoid) {
  66. return uploadedFile;
  67. });
  68. }
  69. });
  70. } else {
  71. // If the file is yet to be processed,
  72. // 1. Remove it from the queue,
  73. // 2. Force upload the current file
  74. // 3. Trigger the callback for the original request
  75. item = _queue.remove(file.generatedID);
  76. return _encryptAndUploadFileToCollection(file, collectionID,
  77. forcedUpload: true)
  78. .then((uploadedFile) {
  79. if (item.collectionID == collectionID) {
  80. item.completer.complete(uploadedFile);
  81. return uploadedFile;
  82. } else {
  83. CollectionsService.instance
  84. .addToCollection(item.collectionID, [uploadedFile]).then((aVoid) {
  85. item.completer.complete(uploadedFile);
  86. });
  87. return uploadedFile;
  88. }
  89. });
  90. }
  91. }
  92. void _pollQueue() {
  93. if (_queue.length > 0 && _currentlyUploading < _maximumConcurrentUploads) {
  94. final firstPendingEntry = _queue.entries
  95. .firstWhere((entry) => entry.value.status == UploadStatus.not_started)
  96. .value;
  97. firstPendingEntry.status = UploadStatus.in_progress;
  98. _encryptAndUploadFileToCollection(
  99. firstPendingEntry.file, firstPendingEntry.collectionID);
  100. }
  101. }
  102. Future<File> _encryptAndUploadFileToCollection(File file, int collectionID,
  103. {bool forcedUpload = false}) async {
  104. _logger.info("Uploading " + file.toString());
  105. if (!forcedUpload) {
  106. _currentlyUploading++;
  107. }
  108. final encryptedFileName = file.generatedID.toString() + ".encrypted";
  109. final tempDirectory = Configuration.instance.getTempDirectory();
  110. final encryptedFilePath = tempDirectory + encryptedFileName;
  111. final sourceFile = (await (await file.getAsset()).originFile);
  112. final encryptedFile = io.File(encryptedFilePath);
  113. final fileAttributes =
  114. await CryptoUtil.encryptFile(sourceFile.path, encryptedFilePath);
  115. final fileUploadURL = await _getUploadURL();
  116. String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
  117. final thumbnailData = (await (await file.getAsset()).thumbDataWithSize(
  118. THUMBNAIL_LARGE_SIZE,
  119. THUMBNAIL_LARGE_SIZE,
  120. quality: 50,
  121. ));
  122. final encryptedThumbnailName =
  123. file.generatedID.toString() + "_thumbnail.encrypted";
  124. final encryptedThumbnailPath = tempDirectory + encryptedThumbnailName;
  125. final encryptedThumbnailData =
  126. CryptoUtil.encryptChaCha(thumbnailData, fileAttributes.key);
  127. final encryptedThumbnail = io.File(encryptedThumbnailPath);
  128. encryptedThumbnail.writeAsBytesSync(encryptedThumbnailData.encryptedData);
  129. final thumbnailUploadURL = await _getUploadURL();
  130. String thumbnailObjectKey =
  131. await _putFile(thumbnailUploadURL, encryptedThumbnail);
  132. // h4ck to fetch location data if missing (thank you Android Q+) lazily only during uploads
  133. if (file.location.latitude == 0 && file.location.longitude == 0) {
  134. final latLong = await (await file.getAsset()).latlngAsync();
  135. file.location = Location(latLong.latitude, latLong.longitude);
  136. }
  137. final encryptedMetadataData = CryptoUtil.encryptChaCha(
  138. utf8.encode(jsonEncode(file.getMetadata())), fileAttributes.key);
  139. final encryptedFileKeyData = CryptoUtil.encryptSync(
  140. fileAttributes.key,
  141. CollectionsService.instance.getCollectionKey(collectionID),
  142. );
  143. final encryptedKey = Sodium.bin2base64(encryptedFileKeyData.encryptedData);
  144. final keyDecryptionNonce = Sodium.bin2base64(encryptedFileKeyData.nonce);
  145. final fileDecryptionHeader = Sodium.bin2base64(fileAttributes.header);
  146. final thumbnailDecryptionHeader =
  147. Sodium.bin2base64(encryptedThumbnailData.header);
  148. final encryptedMetadata =
  149. Sodium.bin2base64(encryptedMetadataData.encryptedData);
  150. final metadataDecryptionHeader =
  151. Sodium.bin2base64(encryptedMetadataData.header);
  152. final data = {
  153. "collectionID": collectionID,
  154. "encryptedKey": encryptedKey,
  155. "keyDecryptionNonce": keyDecryptionNonce,
  156. "file": {
  157. "objectKey": fileObjectKey,
  158. "decryptionHeader": fileDecryptionHeader,
  159. },
  160. "thumbnail": {
  161. "objectKey": thumbnailObjectKey,
  162. "decryptionHeader": thumbnailDecryptionHeader,
  163. },
  164. "metadata": {
  165. "encryptedData": encryptedMetadata,
  166. "decryptionHeader": metadataDecryptionHeader,
  167. }
  168. };
  169. return _dio
  170. .post(
  171. Configuration.instance.getHttpEndpoint() + "/files",
  172. options:
  173. Options(headers: {"X-Auth-Token": Configuration.instance.getToken()}),
  174. data: data,
  175. )
  176. .then((response) async {
  177. encryptedFile.deleteSync();
  178. encryptedThumbnail.deleteSync();
  179. final data = response.data;
  180. file.uploadedFileID = data["id"];
  181. file.collectionID = collectionID;
  182. file.updationTime = data["updationTime"];
  183. file.ownerID = data["ownerID"];
  184. file.encryptedKey = encryptedKey;
  185. file.keyDecryptionNonce = keyDecryptionNonce;
  186. file.fileDecryptionHeader = fileDecryptionHeader;
  187. file.thumbnailDecryptionHeader = thumbnailDecryptionHeader;
  188. file.metadataDecryptionHeader = metadataDecryptionHeader;
  189. if (!forcedUpload) {
  190. _currentlyUploading--;
  191. _queue[file.generatedID].completer.complete(file);
  192. _queue.remove(file.generatedID);
  193. _pollQueue();
  194. }
  195. await FilesDB.instance.update(file);
  196. return file;
  197. });
  198. }
  199. Future<UploadURL> _getUploadURL() {
  200. return Dio()
  201. .get(
  202. Configuration.instance.getHttpEndpoint() + "/files/upload-url",
  203. options: Options(
  204. headers: {"X-Auth-Token": Configuration.instance.getToken()}),
  205. )
  206. .then((response) => UploadURL.fromMap(response.data));
  207. }
  208. Future<String> _putFile(UploadURL uploadURL, io.File file) async {
  209. final fileSize = file.lengthSync().toString();
  210. final startTime = DateTime.now().millisecondsSinceEpoch;
  211. _logger.info("Putting file of size " + fileSize + " to " + uploadURL.url);
  212. return Dio()
  213. .put(uploadURL.url,
  214. data: file.openRead(),
  215. options: Options(headers: {
  216. Headers.contentLengthHeader: await file.length(),
  217. }))
  218. .catchError((e) {
  219. _logger.severe(e);
  220. throw e;
  221. }).then((value) {
  222. _logger.info("Upload speed : " +
  223. (file.lengthSync() /
  224. (DateTime.now().millisecondsSinceEpoch - startTime))
  225. .toString() +
  226. " kilo bytes per second");
  227. return uploadURL.objectKey;
  228. });
  229. }
  230. }
  231. class FileUploadItem {
  232. final File file;
  233. final int collectionID;
  234. final Completer<File> completer;
  235. UploadStatus status;
  236. FileUploadItem(
  237. this.file,
  238. this.collectionID,
  239. this.completer, {
  240. this.status = UploadStatus.not_started,
  241. });
  242. }
  243. enum UploadStatus {
  244. not_started,
  245. in_progress,
  246. completed,
  247. }