Merge pull request #415 from ente-io/reupload_hash_check
[Part-0] Rewrite Sync: Use hash to avoid duplicate uploads
This commit is contained in:
Neeraj Gupta
GitHub
commit
81beff6f1d
11 changed files with 640 additions and 315 deletions
|
|
@ -1,33 +1,57 @@
|
|||
import 'dart:io';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:path/path.dart';
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import 'package:sqflite_migration/sqflite_migration.dart';
|
||||
|
||||
class FilesMigrationDB {
|
||||
class FileUpdationDB {
|
||||
static const _databaseName = "ente.files_migration.db";
|
||||
static const _databaseVersion = 1;
|
||||
static final Logger _logger = Logger((FilesMigrationDB).toString());
|
||||
static final Logger _logger = Logger((FileUpdationDB).toString());
|
||||
|
||||
static const tableName = 're_upload_tracker';
|
||||
|
||||
static const columnLocalID = 'local_id';
|
||||
static const columnReason = 'reason';
|
||||
static const missingLocation = 'missing_location';
|
||||
static const modificationTimeUpdated = 'modificationTimeUpdated';
|
||||
|
||||
Future _onCreate(Database db, int version) async {
|
||||
await db.execute(
|
||||
'''
|
||||
CREATE TABLE $tableName (
|
||||
$columnLocalID TEXT NOT NULL,
|
||||
UNIQUE($columnLocalID)
|
||||
);
|
||||
// SQL code to create the database table
|
||||
static List<String> _createTable() {
|
||||
return [
|
||||
'''
|
||||
CREATE TABLE $tableName (
|
||||
$columnLocalID TEXT NOT NULL,
|
||||
UNIQUE($columnLocalID)
|
||||
);
|
||||
''',
|
||||
);
|
||||
];
|
||||
}
|
||||
|
||||
FilesMigrationDB._privateConstructor();
|
||||
static List<String> addReasonColumn() {
|
||||
return [
|
||||
'''
|
||||
ALTER TABLE $tableName ADD COLUMN $columnReason TEXT;
|
||||
''',
|
||||
'''
|
||||
UPDATE $tableName SET $columnReason = '$missingLocation';
|
||||
''',
|
||||
];
|
||||
}
|
||||
|
||||
static final FilesMigrationDB instance =
|
||||
FilesMigrationDB._privateConstructor();
|
||||
static final initializationScript = [..._createTable()];
|
||||
static final migrationScripts = [
|
||||
...addReasonColumn(),
|
||||
];
|
||||
final dbConfig = MigrationConfig(
|
||||
initializationScript: initializationScript,
|
||||
migrationScripts: migrationScripts,
|
||||
);
|
||||
|
||||
FileUpdationDB._privateConstructor();
|
||||
|
||||
static final FileUpdationDB instance = FileUpdationDB._privateConstructor();
|
||||
|
||||
// only have a single app-wide reference to the database
|
||||
static Future<Database> _dbFuture;
|
||||
|
|
@ -40,13 +64,11 @@ class FilesMigrationDB {
|
|||
|
||||
// this opens the database (and creates it if it doesn't exist)
|
||||
Future<Database> _initDatabase() async {
|
||||
final Directory documentsDirectory = await getApplicationDocumentsDirectory();
|
||||
final Directory documentsDirectory =
|
||||
await getApplicationDocumentsDirectory();
|
||||
final String path = join(documentsDirectory.path, _databaseName);
|
||||
return await openDatabase(
|
||||
path,
|
||||
version: _databaseVersion,
|
||||
onCreate: _onCreate,
|
||||
);
|
||||
debugPrint("DB path " + path);
|
||||
return await openDatabaseWithMigration(path, dbConfig);
|
||||
}
|
||||
|
||||
Future<void> clearTable() async {
|
||||
|
|
@ -54,7 +76,10 @@ class FilesMigrationDB {
|
|||
await db.delete(tableName);
|
||||
}
|
||||
|
||||
Future<void> insertMultiple(List<String> fileLocalIDs) async {
|
||||
Future<void> insertMultiple(
|
||||
List<String> fileLocalIDs,
|
||||
String reason,
|
||||
) async {
|
||||
final startTime = DateTime.now();
|
||||
final db = await instance.database;
|
||||
var batch = db.batch();
|
||||
|
|
@ -67,7 +92,7 @@ class FilesMigrationDB {
|
|||
}
|
||||
batch.insert(
|
||||
tableName,
|
||||
_getRowForReUploadTable(localID),
|
||||
_getRowForReUploadTable(localID, reason),
|
||||
conflictAlgorithm: ConflictAlgorithm.replace,
|
||||
);
|
||||
batchCounter++;
|
||||
|
|
@ -84,22 +109,35 @@ class FilesMigrationDB {
|
|||
);
|
||||
}
|
||||
|
||||
Future<int> deleteByLocalIDs(List<String> localIDs) async {
|
||||
Future<void> deleteByLocalIDs(List<String> localIDs, String reason) async {
|
||||
if (localIDs.isEmpty) {
|
||||
return;
|
||||
}
|
||||
String inParam = "";
|
||||
for (final localID in localIDs) {
|
||||
inParam += "'" + localID + "',";
|
||||
}
|
||||
inParam = inParam.substring(0, inParam.length - 1);
|
||||
final db = await instance.database;
|
||||
return await db.delete(
|
||||
tableName,
|
||||
where: '$columnLocalID IN (${localIDs.join(', ')})',
|
||||
await db.rawQuery(
|
||||
'''
|
||||
DELETE FROM $tableName
|
||||
WHERE $columnLocalID IN ($inParam) AND $columnReason = '$reason';
|
||||
''',
|
||||
);
|
||||
}
|
||||
|
||||
Future<List<String>> getLocalIDsForPotentialReUpload(int limit) async {
|
||||
Future<List<String>> getLocalIDsForPotentialReUpload(
|
||||
int limit,
|
||||
String reason,
|
||||
) async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(tableName, limit: limit);
|
||||
String whereClause = '$columnReason = "$reason"';
|
||||
final rows = await db.query(
|
||||
tableName,
|
||||
limit: limit,
|
||||
where: whereClause,
|
||||
);
|
||||
final result = <String>[];
|
||||
for (final row in rows) {
|
||||
result.add(row[columnLocalID]);
|
||||
|
|
@ -107,10 +145,11 @@ class FilesMigrationDB {
|
|||
return result;
|
||||
}
|
||||
|
||||
Map<String, dynamic> _getRowForReUploadTable(String localID) {
|
||||
Map<String, dynamic> _getRowForReUploadTable(String localID, String reason) {
|
||||
assert(localID != null);
|
||||
final row = <String, dynamic>{};
|
||||
row[columnLocalID] = localID;
|
||||
row[columnReason] = reason;
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ import 'package:photos/models/file_type.dart';
|
|||
import 'package:photos/models/location.dart';
|
||||
import 'package:photos/models/magic_metadata.dart';
|
||||
import 'package:photos/services/feature_flag_service.dart';
|
||||
import 'package:photos/utils/file_uploader_util.dart';
|
||||
import 'package:sqflite/sqflite.dart';
|
||||
import 'package:sqflite_migration/sqflite_migration.dart';
|
||||
|
||||
|
|
@ -848,6 +849,31 @@ class FilesDB {
|
|||
}
|
||||
}
|
||||
|
||||
Future<List<File>> getUploadedFilesWithHashes(
|
||||
FileHashData hashData,
|
||||
FileType fileType,
|
||||
int ownerID,
|
||||
) async {
|
||||
String inParam = "'${hashData.fileHash}'";
|
||||
if (fileType == FileType.livePhoto && hashData.zipHash != null) {
|
||||
inParam += ",'${hashData.zipHash}'";
|
||||
}
|
||||
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(
|
||||
table,
|
||||
where: '($columnUploadedFileID != NULL OR $columnUploadedFileID != -1) '
|
||||
'AND $columnOwnerID = ? AND $columnFileType ='
|
||||
' ? '
|
||||
'AND $columnHash IN ($inParam)',
|
||||
whereArgs: [
|
||||
ownerID,
|
||||
getInt(fileType),
|
||||
],
|
||||
);
|
||||
return _convertToFiles(rows);
|
||||
}
|
||||
|
||||
Future<int> update(File file) async {
|
||||
final db = await instance.database;
|
||||
return await db.update(
|
||||
|
|
@ -877,6 +903,15 @@ class FilesDB {
|
|||
);
|
||||
}
|
||||
|
||||
Future<int> deleteByGeneratedID(int genID) async {
|
||||
final db = await instance.database;
|
||||
return db.delete(
|
||||
table,
|
||||
where: '$columnGeneratedID =?',
|
||||
whereArgs: [genID],
|
||||
);
|
||||
}
|
||||
|
||||
Future<int> deleteMultipleUploadedFiles(List<int> uploadedFileIDs) async {
|
||||
final db = await instance.database;
|
||||
return await db.delete(
|
||||
|
|
@ -1089,27 +1124,11 @@ class FilesDB {
|
|||
return collectionMap.values.toList();
|
||||
}
|
||||
|
||||
Future<File> getLastModifiedFileInCollection(int collectionID) async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.query(
|
||||
table,
|
||||
where: '$columnCollectionID = ?',
|
||||
whereArgs: [collectionID],
|
||||
orderBy: '$columnUpdationTime DESC',
|
||||
limit: 1,
|
||||
);
|
||||
if (rows.isNotEmpty) {
|
||||
return _getFileFromRow(rows[0]);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Future<Map<String, int>> getFileCountInDeviceFolders() async {
|
||||
final db = await instance.database;
|
||||
final rows = await db.rawQuery(
|
||||
'''
|
||||
SELECT COUNT($columnGeneratedID) as count, $columnDeviceFolder
|
||||
SELECT COUNT(DISTINCT($columnLocalID)) as count, $columnDeviceFolder
|
||||
FROM $table
|
||||
WHERE $columnLocalID IS NOT NULL
|
||||
GROUP BY $columnDeviceFolder
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import 'package:photos/services/app_lifecycle_service.dart';
|
|||
import 'package:photos/services/billing_service.dart';
|
||||
import 'package:photos/services/collections_service.dart';
|
||||
import 'package:photos/services/feature_flag_service.dart';
|
||||
import 'package:photos/services/file_migration_service.dart';
|
||||
import 'package:photos/services/local_file_update_service.dart';
|
||||
import 'package:photos/services/local_sync_service.dart';
|
||||
import 'package:photos/services/memories_service.dart';
|
||||
import 'package:photos/services/notification_service.dart';
|
||||
|
|
@ -141,7 +141,7 @@ Future<void> _init(bool isBackground, {String via = ''}) async {
|
|||
await SyncService.instance.init();
|
||||
await MemoriesService.instance.init();
|
||||
await LocalSettings.instance.init();
|
||||
await FileMigrationService.instance.init();
|
||||
await LocalFileUpdateService.instance.init();
|
||||
await SearchService.instance.init();
|
||||
if (Platform.isIOS) {
|
||||
PushService.instance.init().then((_) {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
import 'dart:io' as io;
|
||||
|
||||
import 'package:flutter_sodium/flutter_sodium.dart';
|
||||
import 'package:path/path.dart';
|
||||
import 'package:photo_manager/photo_manager.dart';
|
||||
import 'package:photos/core/configuration.dart';
|
||||
|
|
@ -10,8 +7,8 @@ import 'package:photos/models/file_type.dart';
|
|||
import 'package:photos/models/location.dart';
|
||||
import 'package:photos/models/magic_metadata.dart';
|
||||
import 'package:photos/services/feature_flag_service.dart';
|
||||
import 'package:photos/utils/crypto_util.dart';
|
||||
import 'package:photos/utils/exif_util.dart';
|
||||
import 'package:photos/utils/file_uploader_util.dart';
|
||||
|
||||
class File extends EnteFile {
|
||||
int generatedID;
|
||||
|
|
@ -56,7 +53,9 @@ class File extends EnteFile {
|
|||
|
||||
set pubMagicMetadata(val) => _pubMmd = val;
|
||||
|
||||
static const kCurrentMetadataVersion = 1;
|
||||
// in Version 1, live photo hash is stored as zip's hash.
|
||||
// in V2: LivePhoto hash is stored as imgHash:vidHash
|
||||
static const kCurrentMetadataVersion = 2;
|
||||
|
||||
File();
|
||||
|
||||
|
|
@ -136,10 +135,21 @@ class File extends EnteFile {
|
|||
duration = metadata["duration"] ?? 0;
|
||||
exif = metadata["exif"];
|
||||
hash = metadata["hash"];
|
||||
// handle past live photos upload from web client
|
||||
if (hash == null &&
|
||||
fileType == FileType.livePhoto &&
|
||||
metadata.containsKey('imgHash') &&
|
||||
metadata.containsKey('vidHash')) {
|
||||
// convert to imgHash:vidHash
|
||||
hash =
|
||||
'${metadata['imgHash']}$kLivePhotoHashSeparator${metadata['vidHash']}';
|
||||
}
|
||||
metadataVersion = metadata["version"] ?? 0;
|
||||
}
|
||||
|
||||
Future<Map<String, dynamic>> getMetadataForUpload(io.File sourceFile) async {
|
||||
Future<Map<String, dynamic>> getMetadataForUpload(
|
||||
MediaUploadData mediaUploadData,
|
||||
) async {
|
||||
final asset = await getAsset();
|
||||
// asset can be null for files shared to app
|
||||
if (asset != null) {
|
||||
|
|
@ -149,12 +159,13 @@ class File extends EnteFile {
|
|||
}
|
||||
}
|
||||
if (fileType == FileType.image) {
|
||||
final exifTime = await getCreationTimeFromEXIF(sourceFile);
|
||||
final exifTime =
|
||||
await getCreationTimeFromEXIF(mediaUploadData.sourceFile);
|
||||
if (exifTime != null) {
|
||||
creationTime = exifTime.microsecondsSinceEpoch;
|
||||
}
|
||||
}
|
||||
hash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
|
||||
hash = mediaUploadData.hashData?.fileHash;
|
||||
return getMetadata();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -630,6 +630,51 @@ class CollectionsService {
|
|||
}
|
||||
}
|
||||
|
||||
Future<File> linkLocalFileToExistingUploadedFileInAnotherCollection(
|
||||
int destCollectionID, {
|
||||
@required File localFileToUpload,
|
||||
@required File existingUploadedFile,
|
||||
}) async {
|
||||
final params = <String, dynamic>{};
|
||||
params["collectionID"] = destCollectionID;
|
||||
params["files"] = [];
|
||||
final int uploadedFileID = existingUploadedFile.uploadedFileID;
|
||||
|
||||
// encrypt the fileKey with destination collection's key
|
||||
final fileKey = decryptFileKey(existingUploadedFile);
|
||||
final encryptedKeyData =
|
||||
CryptoUtil.encryptSync(fileKey, getCollectionKey(destCollectionID));
|
||||
|
||||
localFileToUpload.encryptedKey =
|
||||
Sodium.bin2base64(encryptedKeyData.encryptedData);
|
||||
localFileToUpload.keyDecryptionNonce =
|
||||
Sodium.bin2base64(encryptedKeyData.nonce);
|
||||
|
||||
params["files"].add(
|
||||
CollectionFileItem(
|
||||
uploadedFileID,
|
||||
localFileToUpload.encryptedKey,
|
||||
localFileToUpload.keyDecryptionNonce,
|
||||
).toMap(),
|
||||
);
|
||||
|
||||
try {
|
||||
await _dio.post(
|
||||
Configuration.instance.getHttpEndpoint() + "/collections/add-files",
|
||||
data: params,
|
||||
options: Options(
|
||||
headers: {"X-Auth-Token": Configuration.instance.getToken()},
|
||||
),
|
||||
);
|
||||
localFileToUpload.collectionID = destCollectionID;
|
||||
localFileToUpload.uploadedFileID = uploadedFileID;
|
||||
await _filesDB.insertMultiple([localFileToUpload]);
|
||||
return localFileToUpload;
|
||||
} catch (e) {
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> restore(int toCollectionID, List<File> files) async {
|
||||
final params = <String, dynamic>{};
|
||||
params["collectionID"] = toCollectionID;
|
||||
|
|
|
|||
|
|
@ -1,137 +0,0 @@
|
|||
import 'dart:async';
|
||||
import 'dart:core';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:photo_manager/photo_manager.dart';
|
||||
import 'package:photos/db/file_migration_db.dart';
|
||||
import 'package:photos/db/files_db.dart';
|
||||
import 'package:shared_preferences/shared_preferences.dart';
|
||||
|
||||
class FileMigrationService {
|
||||
FilesDB _filesDB;
|
||||
FilesMigrationDB _filesMigrationDB;
|
||||
SharedPreferences _prefs;
|
||||
Logger _logger;
|
||||
static const isLocationMigrationComplete = "fm_isLocationMigrationComplete";
|
||||
static const isLocalImportDone = "fm_IsLocalImportDone";
|
||||
Completer<void> _existingMigration;
|
||||
|
||||
FileMigrationService._privateConstructor() {
|
||||
_logger = Logger((FileMigrationService).toString());
|
||||
_filesDB = FilesDB.instance;
|
||||
_filesMigrationDB = FilesMigrationDB.instance;
|
||||
}
|
||||
|
||||
Future<void> init() async {
|
||||
_prefs = await SharedPreferences.getInstance();
|
||||
}
|
||||
|
||||
static FileMigrationService instance =
|
||||
FileMigrationService._privateConstructor();
|
||||
|
||||
Future<bool> _markLocationMigrationAsCompleted() async {
|
||||
_logger.info('marking migration as completed');
|
||||
return _prefs.setBool(isLocationMigrationComplete, true);
|
||||
}
|
||||
|
||||
bool isLocationMigrationCompleted() {
|
||||
return _prefs.get(isLocationMigrationComplete) ?? false;
|
||||
}
|
||||
|
||||
Future<void> runMigration() async {
|
||||
if (_existingMigration != null) {
|
||||
_logger.info("migration is already in progress, skipping");
|
||||
return _existingMigration.future;
|
||||
}
|
||||
_logger.info("start migration");
|
||||
_existingMigration = Completer<void>();
|
||||
try {
|
||||
await _runMigrationForFilesWithMissingLocation();
|
||||
_existingMigration.complete();
|
||||
_existingMigration = null;
|
||||
} catch (e, s) {
|
||||
_logger.severe('failed to perform migration', e, s);
|
||||
_existingMigration.complete();
|
||||
_existingMigration = null;
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _runMigrationForFilesWithMissingLocation() async {
|
||||
if (!Platform.isAndroid) {
|
||||
return;
|
||||
}
|
||||
// migration only needs to run if Android API Level is 29 or higher
|
||||
final int version = int.parse(await PhotoManager.systemVersion());
|
||||
final bool isMigrationRequired = version >= 29;
|
||||
if (isMigrationRequired) {
|
||||
await _importLocalFilesForMigration();
|
||||
final sTime = DateTime.now().microsecondsSinceEpoch;
|
||||
bool hasData = true;
|
||||
const int limitInBatch = 100;
|
||||
while (hasData) {
|
||||
final localIDsToProcess = await _filesMigrationDB
|
||||
.getLocalIDsForPotentialReUpload(limitInBatch);
|
||||
if (localIDsToProcess.isEmpty) {
|
||||
hasData = false;
|
||||
} else {
|
||||
await _checkAndMarkFilesForReUpload(localIDsToProcess);
|
||||
}
|
||||
}
|
||||
final eTime = DateTime.now().microsecondsSinceEpoch;
|
||||
final d = Duration(microseconds: eTime - sTime);
|
||||
_logger.info(
|
||||
'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds',
|
||||
);
|
||||
}
|
||||
await _markLocationMigrationAsCompleted();
|
||||
}
|
||||
|
||||
Future<void> _checkAndMarkFilesForReUpload(
|
||||
List<String> localIDsToProcess,
|
||||
) async {
|
||||
_logger.info("files to process ${localIDsToProcess.length}");
|
||||
final localIDsWithLocation = <String>[];
|
||||
for (var localID in localIDsToProcess) {
|
||||
bool hasLocation = false;
|
||||
try {
|
||||
final assetEntity = await AssetEntity.fromId(localID);
|
||||
if (assetEntity == null) {
|
||||
continue;
|
||||
}
|
||||
final latLng = await assetEntity.latlngAsync();
|
||||
if ((latLng.longitude ?? 0.0) != 0.0 ||
|
||||
(latLng.longitude ?? 0.0) != 0.0) {
|
||||
_logger.finest(
|
||||
'found lat/long ${latLng.longitude}/${latLng.longitude} for ${assetEntity.title} ${assetEntity.relativePath} with id : $localID',
|
||||
);
|
||||
hasLocation = true;
|
||||
}
|
||||
} catch (e, s) {
|
||||
_logger.severe('failed to get asset entity with id $localID', e, s);
|
||||
}
|
||||
if (hasLocation) {
|
||||
localIDsWithLocation.add(localID);
|
||||
}
|
||||
}
|
||||
_logger.info('marking ${localIDsWithLocation.length} files for re-upload');
|
||||
await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation);
|
||||
await _filesMigrationDB.deleteByLocalIDs(localIDsToProcess);
|
||||
}
|
||||
|
||||
Future<void> _importLocalFilesForMigration() async {
|
||||
if (_prefs.containsKey(isLocalImportDone)) {
|
||||
return;
|
||||
}
|
||||
final sTime = DateTime.now().microsecondsSinceEpoch;
|
||||
_logger.info('importing files without location info');
|
||||
final fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation();
|
||||
await _filesMigrationDB.insertMultiple(fileLocalIDs);
|
||||
final eTime = DateTime.now().microsecondsSinceEpoch;
|
||||
final d = Duration(microseconds: eTime - sTime);
|
||||
_logger.info(
|
||||
'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds',
|
||||
);
|
||||
_prefs.setBool(isLocalImportDone, true);
|
||||
}
|
||||
}
|
||||
240
lib/services/local_file_update_service.dart
Normal file
240
lib/services/local_file_update_service.dart
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
import 'dart:async';
|
||||
import 'dart:core';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:photo_manager/photo_manager.dart';
|
||||
import 'package:photos/db/file_updation_db.dart';
|
||||
import 'package:photos/db/files_db.dart';
|
||||
import 'package:photos/models/file.dart' as ente;
|
||||
import 'package:photos/utils/file_uploader_util.dart';
|
||||
import 'package:shared_preferences/shared_preferences.dart';
|
||||
|
||||
// LocalFileUpdateService tracks all the potential local file IDs which have
|
||||
// changed/modified on the device and needed to be uploaded again.
|
||||
class LocalFileUpdateService {
|
||||
FilesDB _filesDB;
|
||||
FileUpdationDB _fileUpdationDB;
|
||||
SharedPreferences _prefs;
|
||||
Logger _logger;
|
||||
static const isLocationMigrationComplete = "fm_isLocationMigrationComplete";
|
||||
static const isLocalImportDone = "fm_IsLocalImportDone";
|
||||
Completer<void> _existingMigration;
|
||||
|
||||
LocalFileUpdateService._privateConstructor() {
|
||||
_logger = Logger((LocalFileUpdateService).toString());
|
||||
_filesDB = FilesDB.instance;
|
||||
_fileUpdationDB = FileUpdationDB.instance;
|
||||
}
|
||||
|
||||
Future<void> init() async {
|
||||
_prefs = await SharedPreferences.getInstance();
|
||||
}
|
||||
|
||||
static LocalFileUpdateService instance =
|
||||
LocalFileUpdateService._privateConstructor();
|
||||
|
||||
Future<bool> _markLocationMigrationAsCompleted() async {
|
||||
_logger.info('marking migration as completed');
|
||||
return _prefs.setBool(isLocationMigrationComplete, true);
|
||||
}
|
||||
|
||||
bool isLocationMigrationCompleted() {
|
||||
return _prefs.get(isLocationMigrationComplete) ?? false;
|
||||
}
|
||||
|
||||
Future<void> markUpdatedFilesForReUpload() async {
|
||||
if (_existingMigration != null) {
|
||||
_logger.info("migration is already in progress, skipping");
|
||||
return _existingMigration.future;
|
||||
}
|
||||
_existingMigration = Completer<void>();
|
||||
try {
|
||||
if (!isLocationMigrationCompleted() && Platform.isAndroid) {
|
||||
_logger.info("start migration for missing location");
|
||||
await _runMigrationForFilesWithMissingLocation();
|
||||
}
|
||||
await _markFilesWhichAreActuallyUpdated();
|
||||
} catch (e, s) {
|
||||
_logger.severe('failed to perform migration', e, s);
|
||||
} finally {
|
||||
_existingMigration?.complete();
|
||||
_existingMigration = null;
|
||||
}
|
||||
}
|
||||
|
||||
// This method analyses all of local files for which the file
|
||||
// modification/update time was changed. It checks if the existing fileHash
|
||||
// is different from the hash of uploaded file. If fileHash are different,
|
||||
// then it marks the file for file update.
|
||||
Future<void> _markFilesWhichAreActuallyUpdated() async {
|
||||
final sTime = DateTime.now().microsecondsSinceEpoch;
|
||||
bool hasData = true;
|
||||
const int limitInBatch = 100;
|
||||
while (hasData) {
|
||||
final localIDsToProcess =
|
||||
await _fileUpdationDB.getLocalIDsForPotentialReUpload(
|
||||
limitInBatch,
|
||||
FileUpdationDB.modificationTimeUpdated,
|
||||
);
|
||||
if (localIDsToProcess.isEmpty) {
|
||||
hasData = false;
|
||||
} else {
|
||||
await _checkAndMarkFilesWithDifferentHashForFileUpdate(
|
||||
localIDsToProcess,
|
||||
);
|
||||
}
|
||||
}
|
||||
final eTime = DateTime.now().microsecondsSinceEpoch;
|
||||
final d = Duration(microseconds: eTime - sTime);
|
||||
_logger.info(
|
||||
'_markFilesWhichAreActuallyUpdated migration completed in ${d.inSeconds.toString()} seconds',
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> _checkAndMarkFilesWithDifferentHashForFileUpdate(
|
||||
List<String> localIDsToProcess,
|
||||
) async {
|
||||
_logger.info("files to process ${localIDsToProcess.length} for reupload");
|
||||
List<ente.File> localFiles =
|
||||
(await FilesDB.instance.getLocalFiles(localIDsToProcess));
|
||||
Set<String> processedIDs = {};
|
||||
for (ente.File file in localFiles) {
|
||||
if (processedIDs.contains(file.localID)) {
|
||||
continue;
|
||||
}
|
||||
MediaUploadData uploadData;
|
||||
try {
|
||||
uploadData = await getUploadData(file);
|
||||
if (uploadData != null &&
|
||||
uploadData.hashData != null &&
|
||||
file.hash != null &&
|
||||
(file.hash == uploadData.hashData.fileHash ||
|
||||
file.hash == uploadData.hashData.zipHash)) {
|
||||
_logger.info("Skip file update as hash matched ${file.tag()}");
|
||||
} else {
|
||||
_logger.info(
|
||||
"Marking for file update as hash did not match ${file.tag()}",
|
||||
);
|
||||
await FilesDB.instance.updateUploadedFile(
|
||||
file.localID,
|
||||
file.title,
|
||||
file.location,
|
||||
file.creationTime,
|
||||
file.modificationTime,
|
||||
null,
|
||||
);
|
||||
}
|
||||
processedIDs.add(file.localID);
|
||||
} catch (e) {
|
||||
_logger.severe("Failed to get file uploadData", e);
|
||||
} finally {}
|
||||
}
|
||||
debugPrint("Deleting files ${processedIDs.length}");
|
||||
await _fileUpdationDB.deleteByLocalIDs(
|
||||
processedIDs.toList(),
|
||||
FileUpdationDB.modificationTimeUpdated,
|
||||
);
|
||||
}
|
||||
|
||||
Future<MediaUploadData> getUploadData(ente.File file) async {
|
||||
final mediaUploadData = await getUploadDataFromEnteFile(file);
|
||||
// delete the file from app's internal cache if it was copied to app
|
||||
// for upload. Shared Media should only be cleared when the upload
|
||||
// succeeds.
|
||||
if (Platform.isIOS &&
|
||||
mediaUploadData != null &&
|
||||
mediaUploadData.sourceFile != null) {
|
||||
await mediaUploadData.sourceFile.delete();
|
||||
}
|
||||
return mediaUploadData;
|
||||
}
|
||||
|
||||
Future<void> _runMigrationForFilesWithMissingLocation() async {
|
||||
if (!Platform.isAndroid) {
|
||||
return;
|
||||
}
|
||||
// migration only needs to run if Android API Level is 29 or higher
|
||||
final int version = int.parse(await PhotoManager.systemVersion());
|
||||
bool isMigrationRequired = version >= 29;
|
||||
if (isMigrationRequired) {
|
||||
await _importLocalFilesForMigration();
|
||||
final sTime = DateTime.now().microsecondsSinceEpoch;
|
||||
bool hasData = true;
|
||||
const int limitInBatch = 100;
|
||||
while (hasData) {
|
||||
var localIDsToProcess =
|
||||
await _fileUpdationDB.getLocalIDsForPotentialReUpload(
|
||||
limitInBatch,
|
||||
FileUpdationDB.missingLocation,
|
||||
);
|
||||
if (localIDsToProcess.isEmpty) {
|
||||
hasData = false;
|
||||
} else {
|
||||
await _checkAndMarkFilesWithLocationForReUpload(localIDsToProcess);
|
||||
}
|
||||
}
|
||||
final eTime = DateTime.now().microsecondsSinceEpoch;
|
||||
final d = Duration(microseconds: eTime - sTime);
|
||||
_logger.info(
|
||||
'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds',
|
||||
);
|
||||
}
|
||||
await _markLocationMigrationAsCompleted();
|
||||
}
|
||||
|
||||
Future<void> _checkAndMarkFilesWithLocationForReUpload(
|
||||
List<String> localIDsToProcess,
|
||||
) async {
|
||||
_logger.info("files to process ${localIDsToProcess.length}");
|
||||
var localIDsWithLocation = <String>[];
|
||||
for (var localID in localIDsToProcess) {
|
||||
bool hasLocation = false;
|
||||
try {
|
||||
var assetEntity = await AssetEntity.fromId(localID);
|
||||
if (assetEntity == null) {
|
||||
continue;
|
||||
}
|
||||
var latLng = await assetEntity.latlngAsync();
|
||||
if ((latLng.longitude ?? 0.0) != 0.0 ||
|
||||
(latLng.longitude ?? 0.0) != 0.0) {
|
||||
_logger.finest(
|
||||
'found lat/long ${latLng.longitude}/${latLng.longitude} for ${assetEntity.title} ${assetEntity.relativePath} with id : $localID',
|
||||
);
|
||||
hasLocation = true;
|
||||
}
|
||||
} catch (e, s) {
|
||||
_logger.severe('failed to get asset entity with id $localID', e, s);
|
||||
}
|
||||
if (hasLocation) {
|
||||
localIDsWithLocation.add(localID);
|
||||
}
|
||||
}
|
||||
_logger.info('marking ${localIDsWithLocation.length} files for re-upload');
|
||||
await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation);
|
||||
await _fileUpdationDB.deleteByLocalIDs(
|
||||
localIDsToProcess,
|
||||
FileUpdationDB.missingLocation,
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> _importLocalFilesForMigration() async {
|
||||
if (_prefs.containsKey(isLocalImportDone)) {
|
||||
return;
|
||||
}
|
||||
final sTime = DateTime.now().microsecondsSinceEpoch;
|
||||
_logger.info('importing files without location info');
|
||||
var fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation();
|
||||
await _fileUpdationDB.insertMultiple(
|
||||
fileLocalIDs,
|
||||
FileUpdationDB.missingLocation,
|
||||
);
|
||||
final eTime = DateTime.now().microsecondsSinceEpoch;
|
||||
final d = Duration(microseconds: eTime - sTime);
|
||||
_logger.info(
|
||||
'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds',
|
||||
);
|
||||
await _prefs.setBool(isLocalImportDone, true);
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import 'package:logging/logging.dart';
|
|||
import 'package:photo_manager/photo_manager.dart';
|
||||
import 'package:photos/core/configuration.dart';
|
||||
import 'package:photos/core/event_bus.dart';
|
||||
import 'package:photos/db/file_updation_db.dart';
|
||||
import 'package:photos/db/files_db.dart';
|
||||
import 'package:photos/events/local_photos_updated_event.dart';
|
||||
import 'package:photos/events/sync_status_update_event.dart';
|
||||
|
|
@ -31,6 +32,7 @@ class LocalSyncService {
|
|||
// Adding `_2` as a suffic to pull files that were earlier ignored due to permission errors
|
||||
// See https://github.com/CaiJingLong/flutter_photo_manager/issues/589
|
||||
static const kInvalidFileIDsKey = "invalid_file_ids_2";
|
||||
|
||||
LocalSyncService._privateConstructor();
|
||||
|
||||
static final LocalSyncService instance =
|
||||
|
|
@ -243,17 +245,15 @@ class LocalSyncService {
|
|||
updatedFiles.length.toString() + " local files were updated.",
|
||||
);
|
||||
}
|
||||
|
||||
final List<String> updatedLocalIDs = [];
|
||||
for (final file in updatedFiles) {
|
||||
await captureUpdateLogs(file);
|
||||
await _db.updateUploadedFile(
|
||||
file.localID,
|
||||
file.title,
|
||||
file.location,
|
||||
file.creationTime,
|
||||
file.modificationTime,
|
||||
null,
|
||||
);
|
||||
updatedLocalIDs.add(file.localID);
|
||||
}
|
||||
await FileUpdationDB.instance.insertMultiple(
|
||||
updatedLocalIDs,
|
||||
FileUpdationDB.modificationTimeUpdated,
|
||||
);
|
||||
final List<File> allFiles = [];
|
||||
allFiles.addAll(files);
|
||||
files.removeWhere((file) => existingLocalFileIDs.contains(file.localID));
|
||||
|
|
@ -265,32 +265,6 @@ class LocalSyncService {
|
|||
await _prefs.setInt(kDbUpdationTimeKey, toTime);
|
||||
}
|
||||
|
||||
// _captureUpdateLogs is a helper method to log details
|
||||
// about the file which is being marked for re-upload
|
||||
Future<void> captureUpdateLogs(File file) async {
|
||||
_logger.info(
|
||||
're-upload locally updated file ${file.toString()}',
|
||||
);
|
||||
try {
|
||||
if (Platform.isIOS) {
|
||||
final assetEntity = await AssetEntity.fromId(file.localID);
|
||||
if (assetEntity != null) {
|
||||
final isLocallyAvailable =
|
||||
await assetEntity.isLocallyAvailable(isOrigin: true);
|
||||
_logger.info(
|
||||
're-upload asset ${file.toString()} with localAvailableFlag '
|
||||
'$isLocallyAvailable and fav ${assetEntity.isFavorite}',
|
||||
);
|
||||
} else {
|
||||
_logger
|
||||
.info('re-upload failed to fetch assetInfo ${file.toString()}');
|
||||
}
|
||||
}
|
||||
} catch (ignore) {
|
||||
//ignore
|
||||
}
|
||||
}
|
||||
|
||||
void _updatePathsToBackup(List<File> files) {
|
||||
if (Configuration.instance.hasSelectedAllFoldersForBackup()) {
|
||||
final pathsToBackup = Configuration.instance.getPathsToBackUp();
|
||||
|
|
|
|||
|
|
@ -16,9 +16,8 @@ import 'package:photos/models/file.dart';
|
|||
import 'package:photos/models/file_type.dart';
|
||||
import 'package:photos/services/app_lifecycle_service.dart';
|
||||
import 'package:photos/services/collections_service.dart';
|
||||
import 'package:photos/services/feature_flag_service.dart';
|
||||
import 'package:photos/services/file_migration_service.dart';
|
||||
import 'package:photos/services/ignored_files_service.dart';
|
||||
import 'package:photos/services/local_file_update_service.dart';
|
||||
import 'package:photos/services/local_sync_service.dart';
|
||||
import 'package:photos/services/trash_sync_service.dart';
|
||||
import 'package:photos/utils/diff_fetcher.dart';
|
||||
|
|
@ -32,8 +31,8 @@ class RemoteSyncService {
|
|||
final _uploader = FileUploader.instance;
|
||||
final _collectionsService = CollectionsService.instance;
|
||||
final _diffFetcher = DiffFetcher();
|
||||
final FileMigrationService _fileMigrationService =
|
||||
FileMigrationService.instance;
|
||||
final LocalFileUpdateService _localFileUpdateService =
|
||||
LocalFileUpdateService.instance;
|
||||
int _completedUploads = 0;
|
||||
SharedPreferences _prefs;
|
||||
Completer<void> _existingSync;
|
||||
|
|
@ -134,10 +133,8 @@ class RemoteSyncService {
|
|||
if (!_hasReSynced()) {
|
||||
await _markReSyncAsDone();
|
||||
}
|
||||
if (FeatureFlagService.instance.enableMissingLocationMigration() &&
|
||||
!_fileMigrationService.isLocationMigrationCompleted()) {
|
||||
_fileMigrationService.runMigration();
|
||||
}
|
||||
|
||||
unawaited(_localFileUpdateService.markUpdatedFilesForReUpload());
|
||||
}
|
||||
|
||||
Future<void> _syncUpdatedCollections(bool silently) async {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import 'dart:typed_data';
|
|||
|
||||
import 'package:connectivity/connectivity.dart';
|
||||
import 'package:dio/dio.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter_sodium/flutter_sodium.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:path/path.dart';
|
||||
|
|
@ -31,6 +32,7 @@ import 'package:photos/utils/crypto_util.dart';
|
|||
import 'package:photos/utils/file_download_util.dart';
|
||||
import 'package:photos/utils/file_uploader_util.dart';
|
||||
import 'package:shared_preferences/shared_preferences.dart';
|
||||
import 'package:tuple/tuple.dart';
|
||||
|
||||
class FileUploader {
|
||||
static const kMaximumConcurrentUploads = 4;
|
||||
|
|
@ -108,6 +110,8 @@ class FileUploader {
|
|||
});
|
||||
}
|
||||
|
||||
// upload future will return null as File when the file entry is deleted
|
||||
// locally because it's already present in the destination collection.
|
||||
Future<File> upload(File file, int collectionID) {
|
||||
// If the file hasn't been queued yet, queue it
|
||||
_totalCountInUploadSession++;
|
||||
|
|
@ -125,10 +129,25 @@ class FileUploader {
|
|||
_totalCountInUploadSession--;
|
||||
return item.completer.future;
|
||||
}
|
||||
|
||||
debugPrint(
|
||||
"Wait on another upload on same local ID to finish before "
|
||||
"adding it to new collection",
|
||||
);
|
||||
// Else wait for the existing upload to complete,
|
||||
// and add it to the relevant collection
|
||||
return item.completer.future.then((uploadedFile) {
|
||||
// If the fileUploader completer returned null,
|
||||
_logger.info(
|
||||
"original upload completer resolved, try adding the file to another "
|
||||
"collection",
|
||||
);
|
||||
if (uploadedFile == null) {
|
||||
/* todo: handle this case, ideally during next sync the localId
|
||||
should be uploaded to this collection ID
|
||||
*/
|
||||
_logger.severe('unexpected upload state');
|
||||
return null;
|
||||
}
|
||||
return CollectionsService.instance
|
||||
.addToCollection(collectionID, [uploadedFile]).then((aVoid) {
|
||||
return uploadedFile;
|
||||
|
|
@ -136,61 +155,6 @@ class FileUploader {
|
|||
});
|
||||
}
|
||||
|
||||
Future<File> forceUpload(File file, int collectionID) async {
|
||||
_logger.info(
|
||||
"Force uploading " +
|
||||
file.toString() +
|
||||
" into collection " +
|
||||
collectionID.toString(),
|
||||
);
|
||||
_totalCountInUploadSession++;
|
||||
// If the file hasn't been queued yet, ez.
|
||||
if (!_queue.containsKey(file.localID)) {
|
||||
final completer = Completer<File>();
|
||||
_queue[file.localID] = FileUploadItem(
|
||||
file,
|
||||
collectionID,
|
||||
completer,
|
||||
status: UploadStatus.inProgress,
|
||||
);
|
||||
_encryptAndUploadFileToCollection(file, collectionID, forcedUpload: true);
|
||||
return completer.future;
|
||||
}
|
||||
var item = _queue[file.localID];
|
||||
// If the file is being uploaded right now, wait and proceed
|
||||
if (item.status == UploadStatus.inProgress ||
|
||||
item.status == UploadStatus.inBackground) {
|
||||
_totalCountInUploadSession--;
|
||||
final uploadedFile = await item.completer.future;
|
||||
if (uploadedFile.collectionID == collectionID) {
|
||||
// Do nothing
|
||||
} else {
|
||||
await CollectionsService.instance
|
||||
.addToCollection(collectionID, [uploadedFile]);
|
||||
}
|
||||
return uploadedFile;
|
||||
} else {
|
||||
// If the file is yet to be processed,
|
||||
// 1. Set the status to in_progress
|
||||
// 2. Force upload the file
|
||||
// 3. Add to the relevant collection
|
||||
item = _queue[file.localID];
|
||||
item.status = UploadStatus.inProgress;
|
||||
final uploadedFile = await _encryptAndUploadFileToCollection(
|
||||
file,
|
||||
collectionID,
|
||||
forcedUpload: true,
|
||||
);
|
||||
if (item.collectionID == collectionID) {
|
||||
return uploadedFile;
|
||||
} else {
|
||||
await CollectionsService.instance
|
||||
.addToCollection(item.collectionID, [uploadedFile]);
|
||||
return uploadedFile;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int getCurrentSessionUploadCount() {
|
||||
return _totalCountInUploadSession;
|
||||
}
|
||||
|
|
@ -319,6 +283,7 @@ class FileUploader {
|
|||
fileOnDisk.updationTime != -1 &&
|
||||
fileOnDisk.collectionID == collectionID;
|
||||
if (wasAlreadyUploaded) {
|
||||
debugPrint("File is already uploaded ${fileOnDisk.tag()}");
|
||||
return fileOnDisk;
|
||||
}
|
||||
|
||||
|
|
@ -362,6 +327,7 @@ class FileUploader {
|
|||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
Uint8List key;
|
||||
final bool isUpdatedFile =
|
||||
file.uploadedFileID != null && file.updationTime == -1;
|
||||
|
|
@ -370,6 +336,22 @@ class FileUploader {
|
|||
key = decryptFileKey(file);
|
||||
} else {
|
||||
key = null;
|
||||
// check if the file is already uploaded and can be mapped to existing
|
||||
// uploaded file. If map is found, it also returns the corresponding
|
||||
// mapped or update file entry.
|
||||
final result = await _mapToExistingUploadWithSameHash(
|
||||
mediaUploadData,
|
||||
file,
|
||||
collectionID,
|
||||
);
|
||||
final isMappedToExistingUpload = result.item1;
|
||||
if (isMappedToExistingUpload) {
|
||||
debugPrint(
|
||||
"File success mapped to existing uploaded ${file.toString()}",
|
||||
);
|
||||
// return the mapped file
|
||||
return result.item2;
|
||||
}
|
||||
}
|
||||
|
||||
if (io.File(encryptedFilePath).existsSync()) {
|
||||
|
|
@ -399,8 +381,7 @@ class FileUploader {
|
|||
final fileUploadURL = await _getUploadURL();
|
||||
final String fileObjectKey = await _putFile(fileUploadURL, encryptedFile);
|
||||
|
||||
final metadata =
|
||||
await file.getMetadataForUpload(mediaUploadData.sourceFile);
|
||||
final metadata = await file.getMetadataForUpload(mediaUploadData);
|
||||
final encryptedMetadataData = await CryptoUtil.encryptChaCha(
|
||||
utf8.encode(jsonEncode(metadata)),
|
||||
fileAttributes.key,
|
||||
|
|
@ -476,25 +457,143 @@ class FileUploader {
|
|||
}
|
||||
rethrow;
|
||||
} finally {
|
||||
if (mediaUploadData != null && mediaUploadData.sourceFile != null) {
|
||||
// delete the file from app's internal cache if it was copied to app
|
||||
// for upload. Shared Media should only be cleared when the upload
|
||||
// succeeds.
|
||||
if (io.Platform.isIOS ||
|
||||
(uploadCompleted && file.isSharedMediaToAppSandbox())) {
|
||||
await mediaUploadData.sourceFile.delete();
|
||||
}
|
||||
}
|
||||
if (io.File(encryptedFilePath).existsSync()) {
|
||||
await io.File(encryptedFilePath).delete();
|
||||
}
|
||||
if (io.File(encryptedThumbnailPath).existsSync()) {
|
||||
await io.File(encryptedThumbnailPath).delete();
|
||||
}
|
||||
await _uploadLocks.releaseLock(file.localID, _processType.toString());
|
||||
await _onUploadDone(
|
||||
mediaUploadData,
|
||||
uploadCompleted,
|
||||
file,
|
||||
encryptedFilePath,
|
||||
encryptedThumbnailPath,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
_mapToExistingUpload links the fileToUpload with the existing uploaded
|
||||
files. if the link is successful, it returns true otherwise false.
|
||||
When false, we should go ahead and re-upload or update the file.
|
||||
It performs following checks:
|
||||
a) Uploaded file with same localID and destination collection. Delete the
|
||||
fileToUpload entry
|
||||
b) Uploaded file in destination collection but with missing localID.
|
||||
Update the localID for uploadedFile and delete the fileToUpload entry
|
||||
c) A uploaded file exist with same localID but in a different collection.
|
||||
or
|
||||
d) Uploaded file in different collection but missing localID.
|
||||
For both c and d, perform add to collection operation.
|
||||
e) File already exists but different localID. Re-upload
|
||||
In case the existing files already have local identifier, which is
|
||||
different from the {fileToUpload}, then most probably device has
|
||||
duplicate files.
|
||||
*/
|
||||
Future<Tuple2<bool, File>> _mapToExistingUploadWithSameHash(
|
||||
MediaUploadData mediaUploadData,
|
||||
File fileToUpload,
|
||||
int toCollectionID,
|
||||
) async {
|
||||
if (fileToUpload.uploadedFileID != null) {
|
||||
// ideally this should never happen, but because the code below this case
|
||||
// can do unexpected mapping, we are adding this additional check
|
||||
_logger.severe(
|
||||
'Critical: file is already uploaded, skipped mapping',
|
||||
);
|
||||
return Tuple2(false, fileToUpload);
|
||||
}
|
||||
|
||||
final List<File> existingUploadedFiles =
|
||||
await FilesDB.instance.getUploadedFilesWithHashes(
|
||||
mediaUploadData.hashData,
|
||||
fileToUpload.fileType,
|
||||
Configuration.instance.getUserID(),
|
||||
);
|
||||
if (existingUploadedFiles?.isEmpty ?? true) {
|
||||
// continueUploading this file
|
||||
return Tuple2(false, fileToUpload);
|
||||
} else {
|
||||
debugPrint("Found some matches");
|
||||
}
|
||||
// case a
|
||||
final File sameLocalSameCollection = existingUploadedFiles.firstWhere(
|
||||
(e) =>
|
||||
e.collectionID == toCollectionID && e.localID == fileToUpload.localID,
|
||||
orElse: () => null,
|
||||
);
|
||||
if (sameLocalSameCollection != null) {
|
||||
debugPrint(
|
||||
"sameLocalSameCollection: \n toUpload ${fileToUpload.tag()} "
|
||||
"\n existing: ${sameLocalSameCollection.tag()}",
|
||||
);
|
||||
// should delete the fileToUploadEntry
|
||||
await FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID);
|
||||
return Tuple2(true, sameLocalSameCollection);
|
||||
}
|
||||
|
||||
// case b
|
||||
final File fileMissingLocalButSameCollection =
|
||||
existingUploadedFiles.firstWhere(
|
||||
(e) => e.collectionID == toCollectionID && e.localID == null,
|
||||
orElse: () => null,
|
||||
);
|
||||
if (fileMissingLocalButSameCollection != null) {
|
||||
// update the local id of the existing file and delete the fileToUpload
|
||||
// entry
|
||||
debugPrint(
|
||||
"fileMissingLocalButSameCollection: \n toUpload ${fileToUpload.tag()} "
|
||||
"\n existing: ${fileMissingLocalButSameCollection.tag()}",
|
||||
);
|
||||
fileMissingLocalButSameCollection.localID = fileToUpload.localID;
|
||||
await FilesDB.instance.insert(fileMissingLocalButSameCollection);
|
||||
await FilesDB.instance.deleteByGeneratedID(fileToUpload.generatedID);
|
||||
return Tuple2(true, fileMissingLocalButSameCollection);
|
||||
}
|
||||
|
||||
// case c and d
|
||||
final File fileExistsButDifferentCollection =
|
||||
existingUploadedFiles.firstWhere(
|
||||
(e) => e.collectionID != toCollectionID,
|
||||
orElse: () => null,
|
||||
);
|
||||
if (fileExistsButDifferentCollection != null) {
|
||||
debugPrint(
|
||||
"fileExistsButDifferentCollection: \n toUpload ${fileToUpload.tag()} "
|
||||
"\n existing: ${fileExistsButDifferentCollection.tag()}",
|
||||
);
|
||||
final linkedFile = await CollectionsService.instance
|
||||
.linkLocalFileToExistingUploadedFileInAnotherCollection(
|
||||
toCollectionID,
|
||||
localFileToUpload: fileToUpload,
|
||||
existingUploadedFile: fileExistsButDifferentCollection,
|
||||
);
|
||||
return Tuple2(true, linkedFile);
|
||||
}
|
||||
// case e
|
||||
return Tuple2(false, fileToUpload);
|
||||
}
|
||||
|
||||
Future<void> _onUploadDone(
|
||||
MediaUploadData mediaUploadData,
|
||||
bool uploadCompleted,
|
||||
File file,
|
||||
String encryptedFilePath,
|
||||
String encryptedThumbnailPath,
|
||||
) async {
|
||||
if (mediaUploadData != null && mediaUploadData.sourceFile != null) {
|
||||
// delete the file from app's internal cache if it was copied to app
|
||||
// for upload. Shared Media should only be cleared when the upload
|
||||
// succeeds.
|
||||
if (io.Platform.isIOS ||
|
||||
(uploadCompleted && file.isSharedMediaToAppSandbox())) {
|
||||
await mediaUploadData.sourceFile.delete();
|
||||
}
|
||||
}
|
||||
if (io.File(encryptedFilePath).existsSync()) {
|
||||
await io.File(encryptedFilePath).delete();
|
||||
}
|
||||
if (io.File(encryptedThumbnailPath).existsSync()) {
|
||||
await io.File(encryptedThumbnailPath).delete();
|
||||
}
|
||||
await _uploadLocks.releaseLock(file.localID, _processType.toString());
|
||||
}
|
||||
|
||||
Future _onInvalidFileError(File file, InvalidFileError e) async {
|
||||
final String ext = file.title == null ? "no title" : extension(file.title);
|
||||
_logger.severe(
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import 'dart:io' as io;
|
|||
import 'dart:typed_data';
|
||||
|
||||
import 'package:archive/archive_io.dart';
|
||||
import 'package:flutter_sodium/flutter_sodium.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:motionphoto/motionphoto.dart';
|
||||
import 'package:path/path.dart';
|
||||
|
|
@ -14,18 +15,37 @@ import 'package:photos/core/errors.dart';
|
|||
import 'package:photos/models/file.dart' as ente;
|
||||
import 'package:photos/models/file_type.dart';
|
||||
import 'package:photos/models/location.dart';
|
||||
import 'package:photos/utils/crypto_util.dart';
|
||||
import 'package:photos/utils/file_util.dart';
|
||||
import 'package:video_thumbnail/video_thumbnail.dart';
|
||||
|
||||
final _logger = Logger("FileUtil");
|
||||
const kMaximumThumbnailCompressionAttempts = 2;
|
||||
const kLivePhotoHashSeparator = ':';
|
||||
|
||||
class MediaUploadData {
|
||||
final io.File sourceFile;
|
||||
final Uint8List thumbnail;
|
||||
final bool isDeleted;
|
||||
final FileHashData hashData;
|
||||
|
||||
MediaUploadData(this.sourceFile, this.thumbnail, this.isDeleted);
|
||||
MediaUploadData(
|
||||
this.sourceFile,
|
||||
this.thumbnail,
|
||||
this.isDeleted,
|
||||
this.hashData,
|
||||
);
|
||||
}
|
||||
|
||||
class FileHashData {
|
||||
// For livePhotos, the fileHash value will be imageHash:videoHash
|
||||
final String fileHash;
|
||||
|
||||
// zipHash is used to take care of existing live photo uploads from older
|
||||
// mobile clients
|
||||
String zipHash;
|
||||
|
||||
FileHashData(this.fileHash, {this.zipHash});
|
||||
}
|
||||
|
||||
Future<MediaUploadData> getUploadDataFromEnteFile(ente.File file) async {
|
||||
|
|
@ -40,6 +60,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
|
|||
io.File sourceFile;
|
||||
Uint8List thumbnailData;
|
||||
bool isDeleted;
|
||||
String fileHash, zipHash;
|
||||
|
||||
// The timeouts are to safeguard against https://github.com/CaiJingLong/flutter_photo_manager/issues/467
|
||||
final asset = await file
|
||||
|
|
@ -72,6 +93,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
|
|||
|
||||
// h4ck to fetch location data if missing (thank you Android Q+) lazily only during uploads
|
||||
await _decorateEnteFileData(file, asset);
|
||||
fileHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
|
||||
|
||||
if (file.fileType == FileType.livePhoto && io.Platform.isIOS) {
|
||||
final io.File videoUrl = await Motionphoto.getLivePhotoFile(file.localID);
|
||||
|
|
@ -81,6 +103,10 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
|
|||
_logger.severe(errMsg);
|
||||
throw InvalidFileUploadState(errMsg);
|
||||
}
|
||||
String livePhotoVideoHash =
|
||||
Sodium.bin2base64(await CryptoUtil.getHash(videoUrl));
|
||||
// imgHash:vidHash
|
||||
fileHash = '$fileHash$kLivePhotoHashSeparator$livePhotoVideoHash';
|
||||
final tempPath = Configuration.instance.getTempDirectory();
|
||||
// .elp -> ente live photo
|
||||
final livePhotoPath = tempPath + file.generatedID.toString() + ".elp";
|
||||
|
|
@ -96,6 +122,7 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
|
|||
}
|
||||
// new sourceFile which needs to be uploaded
|
||||
sourceFile = io.File(livePhotoPath);
|
||||
zipHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
|
||||
}
|
||||
|
||||
thumbnailData = await asset.thumbnailDataWithSize(
|
||||
|
|
@ -116,7 +143,12 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(ente.File file) async {
|
|||
}
|
||||
|
||||
isDeleted = asset == null || !(await asset.exists);
|
||||
return MediaUploadData(sourceFile, thumbnailData, isDeleted);
|
||||
return MediaUploadData(
|
||||
sourceFile,
|
||||
thumbnailData,
|
||||
isDeleted,
|
||||
FileHashData(fileHash, zipHash: zipHash),
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> _decorateEnteFileData(ente.File file, AssetEntity asset) async {
|
||||
|
|
@ -128,7 +160,7 @@ Future<void> _decorateEnteFileData(ente.File file, AssetEntity asset) async {
|
|||
}
|
||||
|
||||
if (file.title == null || file.title.isEmpty) {
|
||||
_logger.severe("Title was missing");
|
||||
_logger.warning("Title was missing ${file.tag()}");
|
||||
file.title = await asset.titleAsync;
|
||||
}
|
||||
}
|
||||
|
|
@ -145,7 +177,13 @@ Future<MediaUploadData> _getMediaUploadDataFromAppCache(ente.File file) async {
|
|||
}
|
||||
try {
|
||||
thumbnailData = await getThumbnailFromInAppCacheFile(file);
|
||||
return MediaUploadData(sourceFile, thumbnailData, isDeleted);
|
||||
final fileHash = Sodium.bin2base64(await CryptoUtil.getHash(sourceFile));
|
||||
return MediaUploadData(
|
||||
sourceFile,
|
||||
thumbnailData,
|
||||
isDeleted,
|
||||
FileHashData(fileHash),
|
||||
);
|
||||
} catch (e, s) {
|
||||
_logger.severe("failed to generate thumbnail", e, s);
|
||||
throw InvalidFileError(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue