123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288 |
- import 'dart:async';
- import 'dart:collection';
- import 'dart:convert';
- import 'dart:io' as io;
- import 'package:dio/dio.dart';
- import 'package:flutter_sodium/flutter_sodium.dart';
- import 'package:logging/logging.dart';
- import 'package:photos/core/configuration.dart';
- import 'package:photos/core/constants.dart';
- import 'package:photos/db/files_db.dart';
- import 'package:photos/models/file.dart';
- import 'package:photos/models/location.dart';
- import 'package:photos/models/upload_url.dart';
- import 'package:photos/services/collections_service.dart';
- import 'package:photos/utils/crypto_util.dart';
- class FileUploader {
- final _logger = Logger("FileUploader");
- final _dio = Dio();
- final _queue = LinkedHashMap<int, FileUploadItem>();
- final _maximumConcurrentUploads = 4;
- int _currentlyUploading = 0;
- final _uploadURLs = Queue<UploadURL>();
- FileUploader._privateConstructor();
- static FileUploader instance = FileUploader._privateConstructor();
- Future<File> upload(File file, int collectionID) {
- // If the file hasn't been queued yet, queue it
- if (!_queue.containsKey(file.generatedID)) {
- final completer = Completer<File>();
- _queue[file.generatedID] = FileUploadItem(file, collectionID, completer);
- _pollQueue();
- return completer.future;
- }
- // If the file exists in the queue for a matching collectionID,
- // return the existing future
- final item = _queue[file.generatedID];
- if (item.collectionID == collectionID) {
- return item.completer.future;
- }
- // Else wait for the existing upload to complete,
- // and add it to the relevant collection
- return item.completer.future.then((uploadedFile) {
- return CollectionsService.instance
- .addToCollection(collectionID, [uploadedFile]).then((aVoid) {
- return uploadedFile;
- });
- });
- }
- Future<File> forceUpload(File file, int collectionID) async {
- // If the file hasn't been queued yet, ez.
- if (!_queue.containsKey(file.generatedID)) {
- return _encryptAndUploadFileToCollection(file, collectionID,
- forcedUpload: true);
- }
- var item = _queue[file.generatedID];
- // If the file is being uploaded right now, wait and proceed
- if (item.status == UploadStatus.in_progress) {
- return item.completer.future.then((uploadedFile) async {
- if (uploadedFile.collectionID == collectionID) {
- // Do nothing
- return uploadedFile;
- } else {
- return CollectionsService.instance
- .addToCollection(collectionID, [uploadedFile]).then((aVoid) {
- return uploadedFile;
- });
- }
- });
- } else {
- // If the file is yet to be processed,
- // 1. Remove it from the queue,
- // 2. Force upload the current file
- // 3. Trigger the callback for the original request
- item = _queue.remove(file.generatedID);
- return _encryptAndUploadFileToCollection(file, collectionID,
- forcedUpload: true)
- .then((uploadedFile) {
- if (item.collectionID == collectionID) {
- item.completer.complete(uploadedFile);
- return uploadedFile;
- } else {
- CollectionsService.instance
- .addToCollection(item.collectionID, [uploadedFile]).then((aVoid) {
- item.completer.complete(uploadedFile);
- });
- return uploadedFile;
- }
- });
- }
- }
- void _pollQueue() {
- if (_queue.length > 0 && _currentlyUploading < _maximumConcurrentUploads) {
- final firstPendingEntry = _queue.entries
- .firstWhere((entry) => entry.value.status == UploadStatus.not_started)
- .value;
- firstPendingEntry.status = UploadStatus.in_progress;
- _encryptAndUploadFileToCollection(
- firstPendingEntry.file, firstPendingEntry.collectionID);
- }
- }
- Future<File> _encryptAndUploadFileToCollection(File file, int collectionID,
- {bool forcedUpload = false}) async {
- _logger.info("Uploading " + file.toString());
- if (!forcedUpload) {
- _currentlyUploading++;
- }
- final encryptedFileName = file.generatedID.toString() + ".encrypted";
- final tempDirectory = Configuration.instance.getTempDirectory();
- final encryptedFilePath = tempDirectory + encryptedFileName;
- final sourceFile = (await (await file.getAsset()).originFile);
- final encryptedFile = io.File(encryptedFilePath);
- final fileAttributes =
- await CryptoUtil.encryptFile(sourceFile.path, encryptedFilePath);
- final fileUploadURL = await _getUploadURL();
- String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
- final thumbnailData = (await (await file.getAsset()).thumbDataWithSize(
- THUMBNAIL_LARGE_SIZE,
- THUMBNAIL_LARGE_SIZE,
- quality: 50,
- ));
- final encryptedThumbnailName =
- file.generatedID.toString() + "_thumbnail.encrypted";
- final encryptedThumbnailPath = tempDirectory + encryptedThumbnailName;
- final encryptedThumbnailData =
- CryptoUtil.encryptChaCha(thumbnailData, fileAttributes.key);
- final encryptedThumbnail = io.File(encryptedThumbnailPath);
- encryptedThumbnail.writeAsBytesSync(encryptedThumbnailData.encryptedData);
- final thumbnailUploadURL = await _getUploadURL();
- String thumbnailObjectKey =
- await _putFile(thumbnailUploadURL, encryptedThumbnail);
- // h4ck to fetch location data if missing (thank you Android Q+) lazily only during uploads
- if (file.location.latitude == 0 && file.location.longitude == 0) {
- final latLong = await (await file.getAsset()).latlngAsync();
- file.location = Location(latLong.latitude, latLong.longitude);
- }
- final encryptedMetadataData = CryptoUtil.encryptChaCha(
- utf8.encode(jsonEncode(file.getMetadata())), fileAttributes.key);
- final encryptedFileKeyData = CryptoUtil.encryptSync(
- fileAttributes.key,
- CollectionsService.instance.getCollectionKey(collectionID),
- );
- final encryptedKey = Sodium.bin2base64(encryptedFileKeyData.encryptedData);
- final keyDecryptionNonce = Sodium.bin2base64(encryptedFileKeyData.nonce);
- final fileDecryptionHeader = Sodium.bin2base64(fileAttributes.header);
- final thumbnailDecryptionHeader =
- Sodium.bin2base64(encryptedThumbnailData.header);
- final encryptedMetadata =
- Sodium.bin2base64(encryptedMetadataData.encryptedData);
- final metadataDecryptionHeader =
- Sodium.bin2base64(encryptedMetadataData.header);
- final data = {
- "collectionID": collectionID,
- "encryptedKey": encryptedKey,
- "keyDecryptionNonce": keyDecryptionNonce,
- "file": {
- "objectKey": fileObjectKey,
- "decryptionHeader": fileDecryptionHeader,
- },
- "thumbnail": {
- "objectKey": thumbnailObjectKey,
- "decryptionHeader": thumbnailDecryptionHeader,
- },
- "metadata": {
- "encryptedData": encryptedMetadata,
- "decryptionHeader": metadataDecryptionHeader,
- }
- };
- return _dio
- .post(
- Configuration.instance.getHttpEndpoint() + "/files",
- options:
- Options(headers: {"X-Auth-Token": Configuration.instance.getToken()}),
- data: data,
- )
- .then((response) async {
- encryptedFile.deleteSync();
- encryptedThumbnail.deleteSync();
- final data = response.data;
- file.uploadedFileID = data["id"];
- file.collectionID = collectionID;
- file.updationTime = data["updationTime"];
- file.ownerID = data["ownerID"];
- file.encryptedKey = encryptedKey;
- file.keyDecryptionNonce = keyDecryptionNonce;
- file.fileDecryptionHeader = fileDecryptionHeader;
- file.thumbnailDecryptionHeader = thumbnailDecryptionHeader;
- file.metadataDecryptionHeader = metadataDecryptionHeader;
- if (!forcedUpload) {
- _currentlyUploading--;
- _queue[file.generatedID].completer.complete(file);
- _queue.remove(file.generatedID);
- _pollQueue();
- }
- await FilesDB.instance.update(file);
- return file;
- });
- }
- Future<UploadURL> _getUploadURL() async {
- if (_uploadURLs.isEmpty) {
- await _fetchUploadURLs();
- }
- return _uploadURLs.removeFirst();
- }
- Future<void> _uploadURLFetchInProgress;
- Future<void> _fetchUploadURLs() {
- if (_uploadURLFetchInProgress == null) {
- _uploadURLFetchInProgress = Dio()
- .get(
- Configuration.instance.getHttpEndpoint() + "/files/upload-urls",
- queryParameters: {
- "count": 42, // m4gic number
- },
- options: Options(
- headers: {"X-Auth-Token": Configuration.instance.getToken()}),
- )
- .then((response) {
- _uploadURLFetchInProgress = null;
- final urls = (response.data["urls"] as List)
- .map((e) => UploadURL.fromMap(e))
- .toList();
- _uploadURLs.addAll(urls);
- });
- }
- return _uploadURLFetchInProgress;
- }
- Future<String> _putFile(UploadURL uploadURL, io.File file) async {
- final fileSize = file.lengthSync().toString();
- final startTime = DateTime.now().millisecondsSinceEpoch;
- _logger.info("Putting file of size " + fileSize + " to " + uploadURL.url);
- return Dio()
- .put(uploadURL.url,
- data: file.openRead(),
- options: Options(headers: {
- Headers.contentLengthHeader: await file.length(),
- }))
- .catchError((e) {
- _logger.severe(e);
- throw e;
- }).then((value) {
- _logger.info("Upload speed : " +
- (file.lengthSync() /
- (DateTime.now().millisecondsSinceEpoch - startTime))
- .toString() +
- " kilo bytes per second");
- return uploadURL.objectKey;
- });
- }
- }
- class FileUploadItem {
- final File file;
- final int collectionID;
- final Completer<File> completer;
- UploadStatus status;
- FileUploadItem(
- this.file,
- this.collectionID,
- this.completer, {
- this.status = UploadStatus.not_started,
- });
- }
- enum UploadStatus {
- not_started,
- in_progress,
- completed,
- }
|