fix: don't re-encrypt file, add nonce field, upload parts logic
This commit is contained in:
Prateek Sunal
parent
4942724423
commit
46b7dba9e3
3 changed files with 129 additions and 58 deletions
|
|
@ -4,12 +4,14 @@ import 'dart:io';
|
|||
import 'package:path/path.dart';
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import "package:photos/core/constants.dart";
|
||||
import "package:photos/models/encryption_result.dart";
|
||||
import "package:photos/module/upload/model/multipart.dart";
|
||||
import "package:photos/utils/crypto_util.dart";
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import "package:sqflite_migration/sqflite_migration.dart";
|
||||
|
||||
class UploadLocksDB {
|
||||
static const _databaseName = "ente.upload_locks.db";
|
||||
static const _databaseVersion = 1;
|
||||
|
||||
static const _uploadLocksTable = (
|
||||
table: "upload_locks",
|
||||
|
|
@ -26,6 +28,7 @@ class UploadLocksDB {
|
|||
columnEncryptedFilePath: "encrypted_file_path",
|
||||
columnEncryptedFileSize: "encrypted_file_size",
|
||||
columnFileKey: "file_key",
|
||||
columnFileNonce: "file_nonce",
|
||||
columnObjectKey: "object_key",
|
||||
columnCompleteUrl: "complete_url",
|
||||
columnStatus: "status",
|
||||
|
|
@ -41,6 +44,19 @@ class UploadLocksDB {
|
|||
columnPartStatus: "part_status",
|
||||
);
|
||||
|
||||
static final initializationScript = [
|
||||
..._createUploadLocksTable(),
|
||||
];
|
||||
|
||||
static final migrationScripts = [
|
||||
..._createTrackUploadsTable(),
|
||||
];
|
||||
|
||||
final dbConfig = MigrationConfig(
|
||||
initializationScript: initializationScript,
|
||||
migrationScripts: migrationScripts,
|
||||
);
|
||||
|
||||
UploadLocksDB._privateConstructor();
|
||||
static final UploadLocksDB instance = UploadLocksDB._privateConstructor();
|
||||
|
||||
|
|
@ -55,18 +71,11 @@ class UploadLocksDB {
|
|||
await getApplicationDocumentsDirectory();
|
||||
final String path = join(documentsDirectory.path, _databaseName);
|
||||
|
||||
return await openDatabase(
|
||||
path,
|
||||
version: _databaseVersion,
|
||||
onCreate: _onCreate,
|
||||
onOpen: (db) async {
|
||||
await _createTrackUploadsTable(db);
|
||||
},
|
||||
);
|
||||
return await openDatabaseWithMigration(path, dbConfig);
|
||||
}
|
||||
|
||||
Future _onCreate(Database db, int version) async {
|
||||
await db.execute(
|
||||
static List<String> _createUploadLocksTable() {
|
||||
return [
|
||||
'''
|
||||
CREATE TABLE ${_uploadLocksTable.table} (
|
||||
${_uploadLocksTable.columnID} TEXT PRIMARY KEY NOT NULL,
|
||||
|
|
@ -74,23 +83,11 @@ class UploadLocksDB {
|
|||
${_uploadLocksTable.columnTime} TEXT NOT NULL
|
||||
)
|
||||
''',
|
||||
);
|
||||
await _createTrackUploadsTable(db);
|
||||
];
|
||||
}
|
||||
|
||||
Future _createTrackUploadsTable(Database db) async {
|
||||
if ((await db.query(
|
||||
'sqlite_master',
|
||||
where: 'name = ?',
|
||||
whereArgs: [
|
||||
_trackUploadTable.table,
|
||||
],
|
||||
))
|
||||
.isNotEmpty) {
|
||||
return;
|
||||
}
|
||||
|
||||
await db.execute(
|
||||
static List<String> _createTrackUploadsTable() {
|
||||
return [
|
||||
'''
|
||||
CREATE TABLE ${_trackUploadTable.table} (
|
||||
${_trackUploadTable.columnID} INTEGER PRIMARY KEY,
|
||||
|
|
@ -99,14 +96,13 @@ class UploadLocksDB {
|
|||
${_trackUploadTable.columnEncryptedFilePath} TEXT NOT NULL,
|
||||
${_trackUploadTable.columnEncryptedFileSize} INTEGER NOT NULL,
|
||||
${_trackUploadTable.columnFileKey} TEXT NOT NULL,
|
||||
${_trackUploadTable.columnFileNonce} TEXT NOT NULL,
|
||||
${_trackUploadTable.columnObjectKey} TEXT NOT NULL,
|
||||
${_trackUploadTable.columnCompleteUrl} TEXT NOT NULL,
|
||||
${_trackUploadTable.columnStatus} TEXT DEFAULT '${MultipartStatus.pending.name}' NOT NULL,
|
||||
${_trackUploadTable.columnPartSize} INTEGER NOT NULL
|
||||
)
|
||||
''',
|
||||
);
|
||||
await db.execute(
|
||||
'''
|
||||
CREATE TABLE ${_partsTable.table} (
|
||||
${_partsTable.columnObjectKey} TEXT NOT NULL REFERENCES ${_trackUploadTable.table}(${_trackUploadTable.columnObjectKey}) ON DELETE CASCADE,
|
||||
|
|
@ -117,7 +113,7 @@ class UploadLocksDB {
|
|||
PRIMARY KEY (${_partsTable.columnObjectKey}, ${_partsTable.columnPartNumber})
|
||||
)
|
||||
''',
|
||||
);
|
||||
];
|
||||
}
|
||||
|
||||
Future<void> clearTable() async {
|
||||
|
|
@ -193,6 +189,33 @@ class UploadLocksDB {
|
|||
return rows.isNotEmpty;
|
||||
}
|
||||
|
||||
Future<EncryptionResult> getFileEncryptionData(
|
||||
String localId,
|
||||
String fileHash,
|
||||
) async {
|
||||
final db = await instance.database;
|
||||
|
||||
final rows = await db.query(
|
||||
_trackUploadTable.table,
|
||||
where:
|
||||
'${_trackUploadTable.columnLocalID} = ? AND ${_trackUploadTable.columnFileHash} = ?',
|
||||
whereArgs: [localId, fileHash],
|
||||
);
|
||||
|
||||
if (rows.isEmpty) {
|
||||
throw Exception("No cached links found for $localId and $fileHash");
|
||||
}
|
||||
final row = rows.first;
|
||||
|
||||
return EncryptionResult(
|
||||
key:
|
||||
CryptoUtil.base642bin(row[_trackUploadTable.columnFileKey] as String),
|
||||
header: CryptoUtil.base642bin(
|
||||
row[_trackUploadTable.columnFileNonce] as String,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Future<MultipartInfo> getCachedLinks(
|
||||
String localId,
|
||||
String fileHash,
|
||||
|
|
@ -255,6 +278,7 @@ class UploadLocksDB {
|
|||
String encryptedFilePath,
|
||||
int fileSize,
|
||||
String fileKey,
|
||||
String fileNonce,
|
||||
) async {
|
||||
final db = await UploadLocksDB.instance.database;
|
||||
final objectKey = urls.objectKey;
|
||||
|
|
@ -269,6 +293,7 @@ class UploadLocksDB {
|
|||
_trackUploadTable.columnEncryptedFilePath: encryptedFilePath,
|
||||
_trackUploadTable.columnEncryptedFileSize: fileSize,
|
||||
_trackUploadTable.columnFileKey: fileKey,
|
||||
_trackUploadTable.columnFileNonce: fileNonce,
|
||||
_trackUploadTable.columnPartSize: multipartPartSizeForUpload,
|
||||
},
|
||||
);
|
||||
|
|
@ -315,14 +340,14 @@ class UploadLocksDB {
|
|||
await db.update(
|
||||
_trackUploadTable.table,
|
||||
{
|
||||
_trackUploadTable.columnStatus: status,
|
||||
_trackUploadTable.columnStatus: status.name,
|
||||
},
|
||||
where: '${_trackUploadTable.columnObjectKey} = ?',
|
||||
whereArgs: [objectKey],
|
||||
);
|
||||
}
|
||||
|
||||
Future<int> deleteCompletedRecord(
|
||||
Future<int> deleteMultipartTrack(
|
||||
String localId,
|
||||
) async {
|
||||
final db = await instance.database;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import "package:dio/dio.dart";
|
|||
import "package:logging/logging.dart";
|
||||
import "package:photos/core/constants.dart";
|
||||
import "package:photos/db/upload_locks_db.dart";
|
||||
import "package:photos/models/encryption_result.dart";
|
||||
import "package:photos/module/upload/model/multipart.dart";
|
||||
import "package:photos/module/upload/model/xml.dart";
|
||||
import "package:photos/services/feature_flag_service.dart";
|
||||
|
|
@ -24,6 +25,13 @@ class MultiPartUploader {
|
|||
this._featureFlagService,
|
||||
);
|
||||
|
||||
Future<EncryptionResult> getEncryptionResult(
|
||||
String localId,
|
||||
String fileHash,
|
||||
) {
|
||||
return _db.getFileEncryptionData(localId, fileHash);
|
||||
}
|
||||
|
||||
Future<int> calculatePartCount(int fileSize) async {
|
||||
final partCount = (fileSize / multipartPartSizeForUpload).ceil();
|
||||
return partCount;
|
||||
|
|
@ -56,6 +64,7 @@ class MultiPartUploader {
|
|||
String encryptedFilePath,
|
||||
int fileSize,
|
||||
Uint8List fileKey,
|
||||
Uint8List fileNonce,
|
||||
) async {
|
||||
await _db.createTrackUploadsEntry(
|
||||
localId,
|
||||
|
|
@ -64,6 +73,7 @@ class MultiPartUploader {
|
|||
encryptedFilePath,
|
||||
fileSize,
|
||||
CryptoUtil.bin2base64(fileKey),
|
||||
CryptoUtil.bin2base64(fileNonce),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -118,12 +128,17 @@ class MultiPartUploader {
|
|||
final partsLength = partsURLs.length;
|
||||
final etags = partInfo.partETags ?? <int, String>{};
|
||||
|
||||
for (int i = 0; i < partsLength; i++) {
|
||||
if (i < (partUploadStatus?.length ?? 0) &&
|
||||
(partUploadStatus?[i] ?? false)) {
|
||||
continue;
|
||||
}
|
||||
final partSize = partInfo.partSize ?? multipartPartSizeForUpload;
|
||||
int i = 0;
|
||||
final partSize = partInfo.partSize ?? multipartPartSizeForUpload;
|
||||
|
||||
// Go to the first part that is not uploaded
|
||||
while (i < (partUploadStatus?.length ?? 0) &&
|
||||
(partUploadStatus?[i] ?? false)) {
|
||||
i++;
|
||||
}
|
||||
|
||||
// Start parts upload
|
||||
while (i < partsLength) {
|
||||
final partURL = partsURLs[i];
|
||||
final isLastPart = i == partsLength - 1;
|
||||
final fileSize =
|
||||
|
|
@ -151,7 +166,9 @@ class MultiPartUploader {
|
|||
etags[i] = eTag!;
|
||||
|
||||
await _db.updatePartStatus(partInfo.urls.objectKey, i, eTag);
|
||||
i++;
|
||||
}
|
||||
|
||||
await _db.updateTrackUploadStatus(
|
||||
partInfo.urls.objectKey,
|
||||
MultipartStatus.uploaded,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import 'dart:async';
|
|||
import 'dart:collection';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:math';
|
||||
import 'dart:math' as math;
|
||||
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:connectivity_plus/connectivity_plus.dart';
|
||||
|
|
@ -41,7 +41,6 @@ import 'package:photos/utils/file_uploader_util.dart';
|
|||
import "package:photos/utils/file_util.dart";
|
||||
import 'package:shared_preferences/shared_preferences.dart';
|
||||
import 'package:tuple/tuple.dart';
|
||||
import "package:uuid/uuid.dart";
|
||||
|
||||
class FileUploader {
|
||||
static const kMaximumConcurrentUploads = 4;
|
||||
|
|
@ -424,12 +423,19 @@ class FileUploader {
|
|||
}
|
||||
|
||||
final tempDirectory = Configuration.instance.getTempDirectory();
|
||||
final String uniqueID = const Uuid().v4().toString();
|
||||
MediaUploadData? mediaUploadData;
|
||||
mediaUploadData = await getUploadDataFromEnteFile(file);
|
||||
|
||||
final String uniqueID = lockKey +
|
||||
"_" +
|
||||
mediaUploadData.hashData!.fileHash!
|
||||
.replaceAll('+', '')
|
||||
.replaceAll('/', '');
|
||||
|
||||
final encryptedFilePath =
|
||||
'$tempDirectory$kUploadTempPrefix${uniqueID}_file.encrypted';
|
||||
final encryptedThumbnailPath =
|
||||
'$tempDirectory$kUploadTempPrefix${uniqueID}_thumb.encrypted';
|
||||
MediaUploadData? mediaUploadData;
|
||||
var uploadCompleted = false;
|
||||
// This flag is used to decide whether to clear the iOS origin file cache
|
||||
// or not.
|
||||
|
|
@ -443,13 +449,25 @@ class FileUploader {
|
|||
'${isUpdatedFile ? 're-upload' : 'upload'} of ${file.toString()}',
|
||||
);
|
||||
|
||||
mediaUploadData = await getUploadDataFromEnteFile(file);
|
||||
var multipartEntryExists = mediaUploadData.hashData?.fileHash != null &&
|
||||
await _uploadLocks.doesExists(
|
||||
lockKey,
|
||||
mediaUploadData.hashData!.fileHash!,
|
||||
);
|
||||
|
||||
Uint8List? key;
|
||||
EncryptionResult? multipartEncryptionResult;
|
||||
if (isUpdatedFile) {
|
||||
key = getFileKey(file);
|
||||
} else {
|
||||
key = null;
|
||||
multipartEncryptionResult = multipartEntryExists
|
||||
? await _multiPartUploader.getEncryptionResult(
|
||||
lockKey,
|
||||
mediaUploadData.hashData!.fileHash!,
|
||||
)
|
||||
: null;
|
||||
key = multipartEncryptionResult?.key;
|
||||
|
||||
// check if the file is already uploaded and can be mapped to existing
|
||||
// uploaded file. If map is found, it also returns the corresponding
|
||||
// mapped or update file entry.
|
||||
|
|
@ -468,16 +486,30 @@ class FileUploader {
|
|||
}
|
||||
}
|
||||
|
||||
if (File(encryptedFilePath).existsSync()) {
|
||||
final encryptedFileExists = File(encryptedFilePath).existsSync();
|
||||
|
||||
// If the multipart entry exists but the encrypted file doesn't, it means
|
||||
// that we'll have to reupload as the nonce is lost
|
||||
if (multipartEntryExists) {
|
||||
if (!encryptedFileExists) {
|
||||
await _uploadLocks.deleteMultipartTrack(lockKey);
|
||||
multipartEntryExists = false;
|
||||
multipartEncryptionResult = null;
|
||||
}
|
||||
} else if (encryptedFileExists) {
|
||||
// otherwise just delete the file for singlepart upload
|
||||
await File(encryptedFilePath).delete();
|
||||
}
|
||||
await _checkIfWithinStorageLimit(mediaUploadData.sourceFile!);
|
||||
final encryptedFile = File(encryptedFilePath);
|
||||
final EncryptionResult fileAttributes = await CryptoUtil.encryptFile(
|
||||
mediaUploadData.sourceFile!.path,
|
||||
encryptedFilePath,
|
||||
key: key,
|
||||
);
|
||||
|
||||
final EncryptionResult fileAttributes = multipartEncryptionResult ??
|
||||
await CryptoUtil.encryptFile(
|
||||
mediaUploadData.sourceFile!.path,
|
||||
encryptedFilePath,
|
||||
key: key,
|
||||
);
|
||||
|
||||
late final Uint8List? thumbnailData;
|
||||
if (mediaUploadData.thumbnail == null &&
|
||||
file.fileType == FileType.video) {
|
||||
|
|
@ -516,11 +548,7 @@ class FileUploader {
|
|||
final fileUploadURL = await _getUploadURL();
|
||||
fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
|
||||
} else {
|
||||
if (mediaUploadData.hashData?.fileHash != null &&
|
||||
await _uploadLocks.doesExists(
|
||||
lockKey,
|
||||
mediaUploadData.hashData!.fileHash!,
|
||||
)) {
|
||||
if (multipartEntryExists) {
|
||||
fileObjectKey = await _multiPartUploader.putExistingMultipartFile(
|
||||
encryptedFile,
|
||||
lockKey,
|
||||
|
|
@ -536,6 +564,7 @@ class FileUploader {
|
|||
encryptedFilePath,
|
||||
await encryptedFile.length(),
|
||||
fileAttributes.key!,
|
||||
fileAttributes.header!,
|
||||
);
|
||||
fileObjectKey = await _multiPartUploader.putMultipartFile(
|
||||
fileUploadURLs,
|
||||
|
|
@ -546,7 +575,7 @@ class FileUploader {
|
|||
|
||||
final metadata = await file.getMetadataForUpload(mediaUploadData);
|
||||
final encryptedMetadataResult = await CryptoUtil.encryptChaCha(
|
||||
utf8.encode(jsonEncode(metadata)) as Uint8List,
|
||||
utf8.encode(jsonEncode(metadata)),
|
||||
fileAttributes.key!,
|
||||
);
|
||||
final fileDecryptionHeader =
|
||||
|
|
@ -628,7 +657,7 @@ class FileUploader {
|
|||
}
|
||||
await FilesDB.instance.update(remoteFile);
|
||||
}
|
||||
await UploadLocksDB.instance.deleteCompletedRecord(lockKey);
|
||||
await UploadLocksDB.instance.deleteMultipartTrack(lockKey);
|
||||
|
||||
if (!_isBackground) {
|
||||
Bus.instance.fire(
|
||||
|
|
@ -1051,7 +1080,7 @@ class FileUploader {
|
|||
if (_uploadURLs.isEmpty) {
|
||||
// the queue is empty, fetch at least for one file to handle force uploads
|
||||
// that are not in the queue. This is to also avoid
|
||||
await fetchUploadURLs(max(_queue.length, 1));
|
||||
await fetchUploadURLs(math.max(_queue.length, 1));
|
||||
}
|
||||
try {
|
||||
return _uploadURLs.removeFirst();
|
||||
|
|
@ -1073,7 +1102,7 @@ class FileUploader {
|
|||
final response = await _enteDio.get(
|
||||
"/files/upload-urls",
|
||||
queryParameters: {
|
||||
"count": min(42, fileCount * 2), // m4gic number
|
||||
"count": math.min(42, fileCount * 2), // m4gic number
|
||||
},
|
||||
);
|
||||
final urls = (response.data["urls"] as List)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue