file_uploader.dart 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288
  1. import 'dart:async';
  2. import 'dart:collection';
  3. import 'dart:convert';
  4. import 'dart:io' as io;
  5. import 'package:dio/dio.dart';
  6. import 'package:flutter_sodium/flutter_sodium.dart';
  7. import 'package:logging/logging.dart';
  8. import 'package:photos/core/configuration.dart';
  9. import 'package:photos/core/constants.dart';
  10. import 'package:photos/db/files_db.dart';
  11. import 'package:photos/models/file.dart';
  12. import 'package:photos/models/location.dart';
  13. import 'package:photos/models/upload_url.dart';
  14. import 'package:photos/services/collections_service.dart';
  15. import 'package:photos/utils/crypto_util.dart';
  16. class FileUploader {
  17. final _logger = Logger("FileUploader");
  18. final _dio = Dio();
  19. final _queue = LinkedHashMap<int, FileUploadItem>();
  20. final _maximumConcurrentUploads = 4;
  21. int _currentlyUploading = 0;
  22. final _uploadURLs = Queue<UploadURL>();
  23. FileUploader._privateConstructor();
  24. static FileUploader instance = FileUploader._privateConstructor();
  25. Future<File> upload(File file, int collectionID) {
  26. // If the file hasn't been queued yet, queue it
  27. if (!_queue.containsKey(file.generatedID)) {
  28. final completer = Completer<File>();
  29. _queue[file.generatedID] = FileUploadItem(file, collectionID, completer);
  30. _pollQueue();
  31. return completer.future;
  32. }
  33. // If the file exists in the queue for a matching collectionID,
  34. // return the existing future
  35. final item = _queue[file.generatedID];
  36. if (item.collectionID == collectionID) {
  37. return item.completer.future;
  38. }
  39. // Else wait for the existing upload to complete,
  40. // and add it to the relevant collection
  41. return item.completer.future.then((uploadedFile) {
  42. return CollectionsService.instance
  43. .addToCollection(collectionID, [uploadedFile]).then((aVoid) {
  44. return uploadedFile;
  45. });
  46. });
  47. }
  48. Future<File> forceUpload(File file, int collectionID) async {
  49. // If the file hasn't been queued yet, ez.
  50. if (!_queue.containsKey(file.generatedID)) {
  51. return _encryptAndUploadFileToCollection(file, collectionID,
  52. forcedUpload: true);
  53. }
  54. var item = _queue[file.generatedID];
  55. // If the file is being uploaded right now, wait and proceed
  56. if (item.status == UploadStatus.in_progress) {
  57. return item.completer.future.then((uploadedFile) async {
  58. if (uploadedFile.collectionID == collectionID) {
  59. // Do nothing
  60. return uploadedFile;
  61. } else {
  62. return CollectionsService.instance
  63. .addToCollection(collectionID, [uploadedFile]).then((aVoid) {
  64. return uploadedFile;
  65. });
  66. }
  67. });
  68. } else {
  69. // If the file is yet to be processed,
  70. // 1. Remove it from the queue,
  71. // 2. Force upload the current file
  72. // 3. Trigger the callback for the original request
  73. item = _queue.remove(file.generatedID);
  74. return _encryptAndUploadFileToCollection(file, collectionID,
  75. forcedUpload: true)
  76. .then((uploadedFile) {
  77. if (item.collectionID == collectionID) {
  78. item.completer.complete(uploadedFile);
  79. return uploadedFile;
  80. } else {
  81. CollectionsService.instance
  82. .addToCollection(item.collectionID, [uploadedFile]).then((aVoid) {
  83. item.completer.complete(uploadedFile);
  84. });
  85. return uploadedFile;
  86. }
  87. });
  88. }
  89. }
  90. void _pollQueue() {
  91. if (_queue.length > 0 && _currentlyUploading < _maximumConcurrentUploads) {
  92. final firstPendingEntry = _queue.entries
  93. .firstWhere((entry) => entry.value.status == UploadStatus.not_started)
  94. .value;
  95. firstPendingEntry.status = UploadStatus.in_progress;
  96. _encryptAndUploadFileToCollection(
  97. firstPendingEntry.file, firstPendingEntry.collectionID);
  98. }
  99. }
  100. Future<File> _encryptAndUploadFileToCollection(File file, int collectionID,
  101. {bool forcedUpload = false}) async {
  102. _logger.info("Uploading " + file.toString());
  103. if (!forcedUpload) {
  104. _currentlyUploading++;
  105. }
  106. final encryptedFileName = file.generatedID.toString() + ".encrypted";
  107. final tempDirectory = Configuration.instance.getTempDirectory();
  108. final encryptedFilePath = tempDirectory + encryptedFileName;
  109. final sourceFile = (await (await file.getAsset()).originFile);
  110. final encryptedFile = io.File(encryptedFilePath);
  111. final fileAttributes =
  112. await CryptoUtil.encryptFile(sourceFile.path, encryptedFilePath);
  113. final fileUploadURL = await _getUploadURL();
  114. String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
  115. final thumbnailData = (await (await file.getAsset()).thumbDataWithSize(
  116. THUMBNAIL_LARGE_SIZE,
  117. THUMBNAIL_LARGE_SIZE,
  118. quality: 50,
  119. ));
  120. final encryptedThumbnailName =
  121. file.generatedID.toString() + "_thumbnail.encrypted";
  122. final encryptedThumbnailPath = tempDirectory + encryptedThumbnailName;
  123. final encryptedThumbnailData =
  124. CryptoUtil.encryptChaCha(thumbnailData, fileAttributes.key);
  125. final encryptedThumbnail = io.File(encryptedThumbnailPath);
  126. encryptedThumbnail.writeAsBytesSync(encryptedThumbnailData.encryptedData);
  127. final thumbnailUploadURL = await _getUploadURL();
  128. String thumbnailObjectKey =
  129. await _putFile(thumbnailUploadURL, encryptedThumbnail);
  130. // h4ck to fetch location data if missing (thank you Android Q+) lazily only during uploads
  131. if (file.location.latitude == 0 && file.location.longitude == 0) {
  132. final latLong = await (await file.getAsset()).latlngAsync();
  133. file.location = Location(latLong.latitude, latLong.longitude);
  134. }
  135. final encryptedMetadataData = CryptoUtil.encryptChaCha(
  136. utf8.encode(jsonEncode(file.getMetadata())), fileAttributes.key);
  137. final encryptedFileKeyData = CryptoUtil.encryptSync(
  138. fileAttributes.key,
  139. CollectionsService.instance.getCollectionKey(collectionID),
  140. );
  141. final encryptedKey = Sodium.bin2base64(encryptedFileKeyData.encryptedData);
  142. final keyDecryptionNonce = Sodium.bin2base64(encryptedFileKeyData.nonce);
  143. final fileDecryptionHeader = Sodium.bin2base64(fileAttributes.header);
  144. final thumbnailDecryptionHeader =
  145. Sodium.bin2base64(encryptedThumbnailData.header);
  146. final encryptedMetadata =
  147. Sodium.bin2base64(encryptedMetadataData.encryptedData);
  148. final metadataDecryptionHeader =
  149. Sodium.bin2base64(encryptedMetadataData.header);
  150. final data = {
  151. "collectionID": collectionID,
  152. "encryptedKey": encryptedKey,
  153. "keyDecryptionNonce": keyDecryptionNonce,
  154. "file": {
  155. "objectKey": fileObjectKey,
  156. "decryptionHeader": fileDecryptionHeader,
  157. },
  158. "thumbnail": {
  159. "objectKey": thumbnailObjectKey,
  160. "decryptionHeader": thumbnailDecryptionHeader,
  161. },
  162. "metadata": {
  163. "encryptedData": encryptedMetadata,
  164. "decryptionHeader": metadataDecryptionHeader,
  165. }
  166. };
  167. return _dio
  168. .post(
  169. Configuration.instance.getHttpEndpoint() + "/files",
  170. options:
  171. Options(headers: {"X-Auth-Token": Configuration.instance.getToken()}),
  172. data: data,
  173. )
  174. .then((response) async {
  175. encryptedFile.deleteSync();
  176. encryptedThumbnail.deleteSync();
  177. final data = response.data;
  178. file.uploadedFileID = data["id"];
  179. file.collectionID = collectionID;
  180. file.updationTime = data["updationTime"];
  181. file.ownerID = data["ownerID"];
  182. file.encryptedKey = encryptedKey;
  183. file.keyDecryptionNonce = keyDecryptionNonce;
  184. file.fileDecryptionHeader = fileDecryptionHeader;
  185. file.thumbnailDecryptionHeader = thumbnailDecryptionHeader;
  186. file.metadataDecryptionHeader = metadataDecryptionHeader;
  187. if (!forcedUpload) {
  188. _currentlyUploading--;
  189. _queue[file.generatedID].completer.complete(file);
  190. _queue.remove(file.generatedID);
  191. _pollQueue();
  192. }
  193. await FilesDB.instance.update(file);
  194. return file;
  195. });
  196. }
  197. Future<UploadURL> _getUploadURL() async {
  198. if (_uploadURLs.isEmpty) {
  199. await _fetchUploadURLs();
  200. }
  201. return _uploadURLs.removeFirst();
  202. }
  203. Future<void> _uploadURLFetchInProgress;
  204. Future<void> _fetchUploadURLs() {
  205. if (_uploadURLFetchInProgress == null) {
  206. _uploadURLFetchInProgress = Dio()
  207. .get(
  208. Configuration.instance.getHttpEndpoint() + "/files/upload-urls",
  209. queryParameters: {
  210. "count": 42, // m4gic number
  211. },
  212. options: Options(
  213. headers: {"X-Auth-Token": Configuration.instance.getToken()}),
  214. )
  215. .then((response) {
  216. _uploadURLFetchInProgress = null;
  217. final urls = (response.data["urls"] as List)
  218. .map((e) => UploadURL.fromMap(e))
  219. .toList();
  220. _uploadURLs.addAll(urls);
  221. });
  222. }
  223. return _uploadURLFetchInProgress;
  224. }
  225. Future<String> _putFile(UploadURL uploadURL, io.File file) async {
  226. final fileSize = file.lengthSync().toString();
  227. final startTime = DateTime.now().millisecondsSinceEpoch;
  228. _logger.info("Putting file of size " + fileSize + " to " + uploadURL.url);
  229. return Dio()
  230. .put(uploadURL.url,
  231. data: file.openRead(),
  232. options: Options(headers: {
  233. Headers.contentLengthHeader: await file.length(),
  234. }))
  235. .catchError((e) {
  236. _logger.severe(e);
  237. throw e;
  238. }).then((value) {
  239. _logger.info("Upload speed : " +
  240. (file.lengthSync() /
  241. (DateTime.now().millisecondsSinceEpoch - startTime))
  242. .toString() +
  243. " kilo bytes per second");
  244. return uploadURL.objectKey;
  245. });
  246. }
  247. }
  248. class FileUploadItem {
  249. final File file;
  250. final int collectionID;
  251. final Completer<File> completer;
  252. UploadStatus status;
  253. FileUploadItem(
  254. this.file,
  255. this.collectionID,
  256. this.completer, {
  257. this.status = UploadStatus.not_started,
  258. });
  259. }
  260. enum UploadStatus {
  261. not_started,
  262. in_progress,
  263. completed,
  264. }