|
@@ -42,7 +42,7 @@ class FileUploader {
|
|
|
|
|
|
final _logger = Logger("FileUploader");
|
|
final _logger = Logger("FileUploader");
|
|
final _dio = Network.instance.getDio();
|
|
final _dio = Network.instance.getDio();
|
|
- final _queue = LinkedHashMap<String, FileUploadItem>();
|
|
|
|
|
|
+ final LinkedHashMap _queue = LinkedHashMap<String, FileUploadItem>();
|
|
final _uploadLocks = UploadLocksDB.instance;
|
|
final _uploadLocks = UploadLocksDB.instance;
|
|
final kSafeBufferForLockExpiry = Duration(days: 1).inMicroseconds;
|
|
final kSafeBufferForLockExpiry = Duration(days: 1).inMicroseconds;
|
|
final kBGTaskDeathTimeout = Duration(seconds: 5).inMicroseconds;
|
|
final kBGTaskDeathTimeout = Duration(seconds: 5).inMicroseconds;
|
|
@@ -68,19 +68,17 @@ class FileUploader {
|
|
Future<void> init(bool isBackground) async {
|
|
Future<void> init(bool isBackground) async {
|
|
_prefs = await SharedPreferences.getInstance();
|
|
_prefs = await SharedPreferences.getInstance();
|
|
_isBackground = isBackground;
|
|
_isBackground = isBackground;
|
|
- _processType =
|
|
|
|
- isBackground ? ProcessType.background : ProcessType.foreground;
|
|
|
|
|
|
+ _processType = isBackground ? ProcessType.background : ProcessType.foreground;
|
|
final currentTime = DateTime.now().microsecondsSinceEpoch;
|
|
final currentTime = DateTime.now().microsecondsSinceEpoch;
|
|
await _uploadLocks.releaseLocksAcquiredByOwnerBefore(
|
|
await _uploadLocks.releaseLocksAcquiredByOwnerBefore(
|
|
_processType.toString(),
|
|
_processType.toString(),
|
|
currentTime,
|
|
currentTime,
|
|
);
|
|
);
|
|
- await _uploadLocks
|
|
|
|
- .releaseAllLocksAcquiredBefore(currentTime - kSafeBufferForLockExpiry);
|
|
|
|
|
|
+ await _uploadLocks.releaseAllLocksAcquiredBefore(currentTime - kSafeBufferForLockExpiry);
|
|
if (!isBackground) {
|
|
if (!isBackground) {
|
|
await _prefs.reload();
|
|
await _prefs.reload();
|
|
- final isBGTaskDead = (_prefs.getInt(kLastBGTaskHeartBeatTime) ?? 0) <
|
|
|
|
- (currentTime - kBGTaskDeathTimeout);
|
|
|
|
|
|
+ final isBGTaskDead =
|
|
|
|
+ (_prefs.getInt(kLastBGTaskHeartBeatTime) ?? 0) < (currentTime - kBGTaskDeathTimeout);
|
|
if (isBGTaskDead) {
|
|
if (isBGTaskDead) {
|
|
await _uploadLocks.releaseLocksAcquiredByOwnerBefore(
|
|
await _uploadLocks.releaseLocksAcquiredByOwnerBefore(
|
|
ProcessType.background.toString(),
|
|
ProcessType.background.toString(),
|
|
@@ -138,10 +136,7 @@ class FileUploader {
|
|
|
|
|
|
Future<File> forceUpload(File file, int collectionID) async {
|
|
Future<File> forceUpload(File file, int collectionID) async {
|
|
_logger.info(
|
|
_logger.info(
|
|
- "Force uploading " +
|
|
|
|
- file.toString() +
|
|
|
|
- " into collection " +
|
|
|
|
- collectionID.toString(),
|
|
|
|
|
|
+ "Force uploading " + file.toString() + " into collection " + collectionID.toString(),
|
|
);
|
|
);
|
|
_totalCountInUploadSession++;
|
|
_totalCountInUploadSession++;
|
|
// If the file hasn't been queued yet, ez.
|
|
// If the file hasn't been queued yet, ez.
|
|
@@ -151,22 +146,20 @@ class FileUploader {
|
|
file,
|
|
file,
|
|
collectionID,
|
|
collectionID,
|
|
completer,
|
|
completer,
|
|
- status: UploadStatus.in_progress,
|
|
|
|
|
|
+ status: UploadStatus.inProgress,
|
|
);
|
|
);
|
|
_encryptAndUploadFileToCollection(file, collectionID, forcedUpload: true);
|
|
_encryptAndUploadFileToCollection(file, collectionID, forcedUpload: true);
|
|
return completer.future;
|
|
return completer.future;
|
|
}
|
|
}
|
|
var item = _queue[file.localID];
|
|
var item = _queue[file.localID];
|
|
// If the file is being uploaded right now, wait and proceed
|
|
// If the file is being uploaded right now, wait and proceed
|
|
- if (item.status == UploadStatus.in_progress ||
|
|
|
|
- item.status == UploadStatus.in_background) {
|
|
|
|
|
|
+ if (item.status == UploadStatus.inProgress || item.status == UploadStatus.inBackground) {
|
|
_totalCountInUploadSession--;
|
|
_totalCountInUploadSession--;
|
|
final uploadedFile = await item.completer.future;
|
|
final uploadedFile = await item.completer.future;
|
|
if (uploadedFile.collectionID == collectionID) {
|
|
if (uploadedFile.collectionID == collectionID) {
|
|
// Do nothing
|
|
// Do nothing
|
|
} else {
|
|
} else {
|
|
- await CollectionsService.instance
|
|
|
|
- .addToCollection(collectionID, [uploadedFile]);
|
|
|
|
|
|
+ await CollectionsService.instance.addToCollection(collectionID, [uploadedFile]);
|
|
}
|
|
}
|
|
return uploadedFile;
|
|
return uploadedFile;
|
|
} else {
|
|
} else {
|
|
@@ -175,7 +168,7 @@ class FileUploader {
|
|
// 2. Force upload the file
|
|
// 2. Force upload the file
|
|
// 3. Add to the relevant collection
|
|
// 3. Add to the relevant collection
|
|
item = _queue[file.localID];
|
|
item = _queue[file.localID];
|
|
- item.status = UploadStatus.in_progress;
|
|
|
|
|
|
+ item.status = UploadStatus.inProgress;
|
|
final uploadedFile = await _encryptAndUploadFileToCollection(
|
|
final uploadedFile = await _encryptAndUploadFileToCollection(
|
|
file,
|
|
file,
|
|
collectionID,
|
|
collectionID,
|
|
@@ -184,8 +177,7 @@ class FileUploader {
|
|
if (item.collectionID == collectionID) {
|
|
if (item.collectionID == collectionID) {
|
|
return uploadedFile;
|
|
return uploadedFile;
|
|
} else {
|
|
} else {
|
|
- await CollectionsService.instance
|
|
|
|
- .addToCollection(item.collectionID, [uploadedFile]);
|
|
|
|
|
|
+ await CollectionsService.instance.addToCollection(item.collectionID, [uploadedFile]);
|
|
return uploadedFile;
|
|
return uploadedFile;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -198,7 +190,7 @@ class FileUploader {
|
|
void clearQueue(final Error reason) {
|
|
void clearQueue(final Error reason) {
|
|
final List<String> uploadsToBeRemoved = [];
|
|
final List<String> uploadsToBeRemoved = [];
|
|
_queue.entries
|
|
_queue.entries
|
|
- .where((entry) => entry.value.status == UploadStatus.not_started)
|
|
|
|
|
|
+ .where((entry) => entry.value.status == UploadStatus.notStarted)
|
|
.forEach((pendingUpload) {
|
|
.forEach((pendingUpload) {
|
|
uploadsToBeRemoved.add(pendingUpload.key);
|
|
uploadsToBeRemoved.add(pendingUpload.key);
|
|
});
|
|
});
|
|
@@ -211,7 +203,7 @@ class FileUploader {
|
|
void removeFromQueueWhere(final bool Function(File) fn, final Error reason) {
|
|
void removeFromQueueWhere(final bool Function(File) fn, final Error reason) {
|
|
List<String> uploadsToBeRemoved = [];
|
|
List<String> uploadsToBeRemoved = [];
|
|
_queue.entries
|
|
_queue.entries
|
|
- .where((entry) => entry.value.status == UploadStatus.not_started)
|
|
|
|
|
|
+ .where((entry) => entry.value.status == UploadStatus.notStarted)
|
|
.forEach((pendingUpload) {
|
|
.forEach((pendingUpload) {
|
|
if (fn(pendingUpload.value.file)) {
|
|
if (fn(pendingUpload.value.file)) {
|
|
uploadsToBeRemoved.add(pendingUpload.key);
|
|
uploadsToBeRemoved.add(pendingUpload.key);
|
|
@@ -235,7 +227,7 @@ class FileUploader {
|
|
if (_uploadCounter < kMaximumConcurrentUploads) {
|
|
if (_uploadCounter < kMaximumConcurrentUploads) {
|
|
var pendingEntry = _queue.entries
|
|
var pendingEntry = _queue.entries
|
|
.firstWhere(
|
|
.firstWhere(
|
|
- (entry) => entry.value.status == UploadStatus.not_started,
|
|
|
|
|
|
+ (entry) => entry.value.status == UploadStatus.notStarted,
|
|
orElse: () => null,
|
|
orElse: () => null,
|
|
)
|
|
)
|
|
?.value;
|
|
?.value;
|
|
@@ -247,14 +239,14 @@ class FileUploader {
|
|
pendingEntry = _queue.entries
|
|
pendingEntry = _queue.entries
|
|
.firstWhere(
|
|
.firstWhere(
|
|
(entry) =>
|
|
(entry) =>
|
|
- entry.value.status == UploadStatus.not_started &&
|
|
|
|
|
|
+ entry.value.status == UploadStatus.notStarted &&
|
|
entry.value.file.fileType != FileType.video,
|
|
entry.value.file.fileType != FileType.video,
|
|
orElse: () => null,
|
|
orElse: () => null,
|
|
)
|
|
)
|
|
?.value;
|
|
?.value;
|
|
}
|
|
}
|
|
if (pendingEntry != null) {
|
|
if (pendingEntry != null) {
|
|
- pendingEntry.status = UploadStatus.in_progress;
|
|
|
|
|
|
+ pendingEntry.status = UploadStatus.inProgress;
|
|
_encryptAndUploadFileToCollection(
|
|
_encryptAndUploadFileToCollection(
|
|
pendingEntry.file,
|
|
pendingEntry.file,
|
|
pendingEntry.collectionID,
|
|
pendingEntry.collectionID,
|
|
@@ -274,8 +266,7 @@ class FileUploader {
|
|
}
|
|
}
|
|
final localID = file.localID;
|
|
final localID = file.localID;
|
|
try {
|
|
try {
|
|
- final uploadedFile =
|
|
|
|
- await _tryToUpload(file, collectionID, forcedUpload).timeout(
|
|
|
|
|
|
+ final uploadedFile = await _tryToUpload(file, collectionID, forcedUpload).timeout(
|
|
kFileUploadTimeout,
|
|
kFileUploadTimeout,
|
|
onTimeout: () {
|
|
onTimeout: () {
|
|
final message = "Upload timed out for file " + file.toString();
|
|
final message = "Upload timed out for file " + file.toString();
|
|
@@ -287,7 +278,7 @@ class FileUploader {
|
|
return uploadedFile;
|
|
return uploadedFile;
|
|
} catch (e) {
|
|
} catch (e) {
|
|
if (e is LockAlreadyAcquiredError) {
|
|
if (e is LockAlreadyAcquiredError) {
|
|
- _queue[localID].status = UploadStatus.in_background;
|
|
|
|
|
|
+ _queue[localID].status = UploadStatus.inBackground;
|
|
return _queue[localID].completer.future;
|
|
return _queue[localID].completer.future;
|
|
} else {
|
|
} else {
|
|
_queue.remove(localID).completer.completeError(e);
|
|
_queue.remove(localID).completer.completeError(e);
|
|
@@ -308,9 +299,8 @@ class FileUploader {
|
|
bool forcedUpload,
|
|
bool forcedUpload,
|
|
) async {
|
|
) async {
|
|
final connectivityResult = await (Connectivity().checkConnectivity());
|
|
final connectivityResult = await (Connectivity().checkConnectivity());
|
|
- var canUploadUnderCurrentNetworkConditions =
|
|
|
|
- (connectivityResult == ConnectivityResult.wifi ||
|
|
|
|
- Configuration.instance.shouldBackupOverMobileData());
|
|
|
|
|
|
+ var canUploadUnderCurrentNetworkConditions = (connectivityResult == ConnectivityResult.wifi ||
|
|
|
|
+ Configuration.instance.shouldBackupOverMobileData());
|
|
if (!canUploadUnderCurrentNetworkConditions && !forcedUpload) {
|
|
if (!canUploadUnderCurrentNetworkConditions && !forcedUpload) {
|
|
throw WiFiUnavailableError();
|
|
throw WiFiUnavailableError();
|
|
}
|
|
}
|
|
@@ -334,10 +324,8 @@ class FileUploader {
|
|
}
|
|
}
|
|
|
|
|
|
final tempDirectory = Configuration.instance.getTempDirectory();
|
|
final tempDirectory = Configuration.instance.getTempDirectory();
|
|
- final encryptedFilePath = tempDirectory +
|
|
|
|
- file.generatedID.toString() +
|
|
|
|
- (_isBackground ? "_bg" : "") +
|
|
|
|
- ".encrypted";
|
|
|
|
|
|
+ final encryptedFilePath =
|
|
|
|
+ tempDirectory + file.generatedID.toString() + (_isBackground ? "_bg" : "") + ".encrypted";
|
|
final encryptedThumbnailPath = tempDirectory +
|
|
final encryptedThumbnailPath = tempDirectory +
|
|
file.generatedID.toString() +
|
|
file.generatedID.toString() +
|
|
"_thumbnail" +
|
|
"_thumbnail" +
|
|
@@ -347,10 +335,7 @@ class FileUploader {
|
|
|
|
|
|
try {
|
|
try {
|
|
_logger.info(
|
|
_logger.info(
|
|
- "Trying to upload " +
|
|
|
|
- file.toString() +
|
|
|
|
- ", isForced: " +
|
|
|
|
- forcedUpload.toString(),
|
|
|
|
|
|
+ "Trying to upload " + file.toString() + ", isForced: " + forcedUpload.toString(),
|
|
);
|
|
);
|
|
try {
|
|
try {
|
|
mediaUploadData = await getUploadDataFromEnteFile(file);
|
|
mediaUploadData = await getUploadDataFromEnteFile(file);
|
|
@@ -362,8 +347,7 @@ class FileUploader {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Uint8List key;
|
|
Uint8List key;
|
|
- bool isUpdatedFile =
|
|
|
|
- file.uploadedFileID != null && file.updationTime == -1;
|
|
|
|
|
|
+ bool isUpdatedFile = file.uploadedFileID != null && file.updationTime == -1;
|
|
if (isUpdatedFile) {
|
|
if (isUpdatedFile) {
|
|
_logger.info("File was updated " + file.toString());
|
|
_logger.info("File was updated " + file.toString());
|
|
key = decryptFileKey(file);
|
|
key = decryptFileKey(file);
|
|
@@ -388,29 +372,23 @@ class FileUploader {
|
|
await io.File(encryptedThumbnailPath).delete();
|
|
await io.File(encryptedThumbnailPath).delete();
|
|
}
|
|
}
|
|
final encryptedThumbnailFile = io.File(encryptedThumbnailPath);
|
|
final encryptedThumbnailFile = io.File(encryptedThumbnailPath);
|
|
- await encryptedThumbnailFile
|
|
|
|
- .writeAsBytes(encryptedThumbnailData.encryptedData);
|
|
|
|
|
|
+ await encryptedThumbnailFile.writeAsBytes(encryptedThumbnailData.encryptedData);
|
|
|
|
|
|
final thumbnailUploadURL = await _getUploadURL();
|
|
final thumbnailUploadURL = await _getUploadURL();
|
|
- String thumbnailObjectKey =
|
|
|
|
- await _putFile(thumbnailUploadURL, encryptedThumbnailFile);
|
|
|
|
|
|
+ String thumbnailObjectKey = await _putFile(thumbnailUploadURL, encryptedThumbnailFile);
|
|
|
|
|
|
final fileUploadURL = await _getUploadURL();
|
|
final fileUploadURL = await _getUploadURL();
|
|
String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
|
|
String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
|
|
|
|
|
|
- final metadata =
|
|
|
|
- await file.getMetadataForUpload(mediaUploadData.sourceFile);
|
|
|
|
|
|
+ final metadata = await file.getMetadataForUpload(mediaUploadData.sourceFile);
|
|
final encryptedMetadataData = await CryptoUtil.encryptChaCha(
|
|
final encryptedMetadataData = await CryptoUtil.encryptChaCha(
|
|
utf8.encode(jsonEncode(metadata)),
|
|
utf8.encode(jsonEncode(metadata)),
|
|
fileAttributes.key,
|
|
fileAttributes.key,
|
|
);
|
|
);
|
|
final fileDecryptionHeader = Sodium.bin2base64(fileAttributes.header);
|
|
final fileDecryptionHeader = Sodium.bin2base64(fileAttributes.header);
|
|
- final thumbnailDecryptionHeader =
|
|
|
|
- Sodium.bin2base64(encryptedThumbnailData.header);
|
|
|
|
- final encryptedMetadata =
|
|
|
|
- Sodium.bin2base64(encryptedMetadataData.encryptedData);
|
|
|
|
- final metadataDecryptionHeader =
|
|
|
|
- Sodium.bin2base64(encryptedMetadataData.header);
|
|
|
|
|
|
+ final thumbnailDecryptionHeader = Sodium.bin2base64(encryptedThumbnailData.header);
|
|
|
|
+ final encryptedMetadata = Sodium.bin2base64(encryptedMetadataData.encryptedData);
|
|
|
|
+ final metadataDecryptionHeader = Sodium.bin2base64(encryptedMetadataData.header);
|
|
if (SyncService.instance.shouldStopSync()) {
|
|
if (SyncService.instance.shouldStopSync()) {
|
|
throw SyncStopRequestedError();
|
|
throw SyncStopRequestedError();
|
|
}
|
|
}
|
|
@@ -434,10 +412,8 @@ class FileUploader {
|
|
fileAttributes.key,
|
|
fileAttributes.key,
|
|
CollectionsService.instance.getCollectionKey(collectionID),
|
|
CollectionsService.instance.getCollectionKey(collectionID),
|
|
);
|
|
);
|
|
- final encryptedKey =
|
|
|
|
- Sodium.bin2base64(encryptedFileKeyData.encryptedData);
|
|
|
|
- final keyDecryptionNonce =
|
|
|
|
- Sodium.bin2base64(encryptedFileKeyData.nonce);
|
|
|
|
|
|
+ final encryptedKey = Sodium.bin2base64(encryptedFileKeyData.encryptedData);
|
|
|
|
+ final keyDecryptionNonce = Sodium.bin2base64(encryptedFileKeyData.nonce);
|
|
remoteFile = await _uploadFile(
|
|
remoteFile = await _uploadFile(
|
|
file,
|
|
file,
|
|
collectionID,
|
|
collectionID,
|
|
@@ -474,9 +450,7 @@ class FileUploader {
|
|
}
|
|
}
|
|
rethrow;
|
|
rethrow;
|
|
} finally {
|
|
} finally {
|
|
- if (io.Platform.isIOS &&
|
|
|
|
- mediaUploadData != null &&
|
|
|
|
- mediaUploadData.sourceFile != null) {
|
|
|
|
|
|
+ if (io.Platform.isIOS && mediaUploadData != null && mediaUploadData.sourceFile != null) {
|
|
await mediaUploadData.sourceFile.delete();
|
|
await mediaUploadData.sourceFile.delete();
|
|
}
|
|
}
|
|
if (io.File(encryptedFilePath).existsSync()) {
|
|
if (io.File(encryptedFilePath).existsSync()) {
|
|
@@ -680,9 +654,7 @@ class FileUploader {
|
|
headers: {"X-Auth-Token": Configuration.instance.getToken()},
|
|
headers: {"X-Auth-Token": Configuration.instance.getToken()},
|
|
),
|
|
),
|
|
);
|
|
);
|
|
- final urls = (response.data["urls"] as List)
|
|
|
|
- .map((e) => UploadURL.fromMap(e))
|
|
|
|
- .toList();
|
|
|
|
|
|
+ final urls = (response.data["urls"] as List).map((e) => UploadURL.fromMap(e)).toList();
|
|
_uploadURLs.addAll(urls);
|
|
_uploadURLs.addAll(urls);
|
|
} on DioError catch (e, s) {
|
|
} on DioError catch (e, s) {
|
|
if (e.response != null) {
|
|
if (e.response != null) {
|
|
@@ -719,10 +691,7 @@ class FileUploader {
|
|
}) async {
|
|
}) async {
|
|
final fileSize = contentLength ?? await file.length();
|
|
final fileSize = contentLength ?? await file.length();
|
|
_logger.info(
|
|
_logger.info(
|
|
- "Putting object for " +
|
|
|
|
- file.toString() +
|
|
|
|
- " of size: " +
|
|
|
|
- fileSize.toString(),
|
|
|
|
|
|
+ "Putting object for " + file.toString() + " of size: " + fileSize.toString(),
|
|
);
|
|
);
|
|
final startTime = DateTime.now().millisecondsSinceEpoch;
|
|
final startTime = DateTime.now().millisecondsSinceEpoch;
|
|
try {
|
|
try {
|
|
@@ -737,8 +706,7 @@ class FileUploader {
|
|
);
|
|
);
|
|
_logger.info(
|
|
_logger.info(
|
|
"Upload speed : " +
|
|
"Upload speed : " +
|
|
- (fileSize / (DateTime.now().millisecondsSinceEpoch - startTime))
|
|
|
|
- .toString() +
|
|
|
|
|
|
+ (fileSize / (DateTime.now().millisecondsSinceEpoch - startTime)).toString() +
|
|
" kilo bytes per second",
|
|
" kilo bytes per second",
|
|
);
|
|
);
|
|
|
|
|
|
@@ -773,9 +741,8 @@ class FileUploader {
|
|
}
|
|
}
|
|
|
|
|
|
Future<void> _pollBackgroundUploadStatus() async {
|
|
Future<void> _pollBackgroundUploadStatus() async {
|
|
- final blockedUploads = _queue.entries
|
|
|
|
- .where((e) => e.value.status == UploadStatus.in_background)
|
|
|
|
- .toList();
|
|
|
|
|
|
+ final blockedUploads =
|
|
|
|
+ _queue.entries.where((e) => e.value.status == UploadStatus.inBackground).toList();
|
|
for (final upload in blockedUploads) {
|
|
for (final upload in blockedUploads) {
|
|
final file = upload.value.file;
|
|
final file = upload.value.file;
|
|
final isStillLocked = await _uploadLocks.isLocked(
|
|
final isStillLocked = await _uploadLocks.isLocked(
|
|
@@ -784,8 +751,7 @@ class FileUploader {
|
|
);
|
|
);
|
|
if (!isStillLocked) {
|
|
if (!isStillLocked) {
|
|
final completer = _queue.remove(upload.key).completer;
|
|
final completer = _queue.remove(upload.key).completer;
|
|
- final dbFile =
|
|
|
|
- await FilesDB.instance.getFile(upload.value.file.generatedID);
|
|
|
|
|
|
+ final dbFile = await FilesDB.instance.getFile(upload.value.file.generatedID);
|
|
if (dbFile.uploadedFileID != null) {
|
|
if (dbFile.uploadedFileID != null) {
|
|
_logger.info("Background upload success detected");
|
|
_logger.info("Background upload success detected");
|
|
completer.complete(dbFile);
|
|
completer.complete(dbFile);
|
|
@@ -811,14 +777,14 @@ class FileUploadItem {
|
|
this.file,
|
|
this.file,
|
|
this.collectionID,
|
|
this.collectionID,
|
|
this.completer, {
|
|
this.completer, {
|
|
- this.status = UploadStatus.not_started,
|
|
|
|
|
|
+ this.status = UploadStatus.notStarted,
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
|
|
enum UploadStatus {
|
|
enum UploadStatus {
|
|
- not_started,
|
|
|
|
- in_progress,
|
|
|
|
- in_background,
|
|
|
|
|
|
+ notStarted,
|
|
|
|
+ inProgress,
|
|
|
|
+ inBackground,
|
|
completed,
|
|
completed,
|
|
}
|
|
}
|
|
|
|
|