0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Move deduplication to a separate service

Browse source
This commit is contained in:
vishnukvmd 2021-09-16 00:31:19 +05:30
parent 8fa782d300
commit 073d07f9d1
3 changed files with 97 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
lib/services/deduplication_service.dart Normal file
View file

@ -0,0 +1,94 @@
import 'package:dio/dio.dart';
import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/core/errors.dart';
import 'package:photos/core/network.dart';
import 'package:photos/db/files_db.dart';
import 'package:photos/models/duplicate_files.dart';
import 'package:photos/models/file.dart';
class DeduplicationService {
final _logger = Logger("DeduplicationService");
final _dio = Network.instance.getDio();
DeduplicationService._privateConstructor();
static final DeduplicationService instance =
DeduplicationService._privateConstructor();
Future<List<DuplicateFiles>> getDuplicateFiles() async {
try {
DuplicateFilesResponse dupes = await _fetchDuplicateFileIDs();
final ids = <int>[];
for (final dupe in dupes.duplicates) {
ids.addAll(dupe.fileIDs);
}
final fileMap = await FilesDB.instance.getFilesFromIDs(ids);
final result = _computeDuplicates(dupes, fileMap);
result.sort((a, b) {
final aSize = a.files.length * a.size;
final bSize = b.files.length * b.size;
return bSize - aSize;
});
return result;
} catch (e) {
_logger.severe(e);
rethrow;
}
}
List<DuplicateFiles> _computeDuplicates(
DuplicateFilesResponse dupes, Map<int, File> fileMap) {
final result = <DuplicateFiles>[];
for (final dupe in dupes.duplicates) {
final files = <File>[];
final Map<int, int> creationTimeCounter = {};
int mostFrequentCreationTime = 0, mostFrequentCreationTimeCount = 0;
for (final id in dupe.fileIDs) {
final file = fileMap[id];
if (file != null) {
if (creationTimeCounter.containsKey(file.creationTime)) {
creationTimeCounter[file.creationTime]++;
} else {
creationTimeCounter[file.creationTime] = 0;
}
if (creationTimeCounter[file.creationTime] >
mostFrequentCreationTimeCount) {
mostFrequentCreationTimeCount =
creationTimeCounter[file.creationTime];
mostFrequentCreationTime = file.creationTime;
}
files.add(file);
} else {
_logger.severe(
"Missing file",
InvalidStateError(
"Could not find file in local DB " + id.toString()));
}
}
final incorrectDuplicates = <File>{};
for (final file in files) {
if (file.creationTime != mostFrequentCreationTime) {
incorrectDuplicates.add(file);
}
}
files.removeWhere((file) => incorrectDuplicates.contains(file));
if (files.length > 1) {
result.add(DuplicateFiles(files, dupe.size));
}
}
return result;
}
Future<DuplicateFilesResponse> _fetchDuplicateFileIDs() async {
final response = await _dio.get(
Configuration.instance.getHttpEndpoint() + "/files/duplicates",
options: Options(
headers: {
"X-Auth-Token": Configuration.instance.getToken(),
},
),
);
return DuplicateFilesResponse.fromMap(response.data);
}
}

68
lib/services/sync_service.dart
View file

@ -16,8 +16,6 @@ import 'package:photos/events/subscription_purchased_event.dart';
import 'package:photos/events/sync_status_update_event.dart';
import 'package:photos/events/trigger_logout_event.dart';
import 'package:photos/models/backup_status.dart';
import 'package:photos/models/duplicate_files.dart';
import 'package:photos/models/file.dart';
import 'package:photos/models/file_type.dart';
import 'package:photos/services/local_sync_service.dart';
import 'package:photos/services/notification_service.dart';
@ -185,72 +183,6 @@ class SyncService {
);
}
Future<List<DuplicateFiles>> getDuplicateFiles() async {
try {
final response = await _dio.get(
Configuration.instance.getHttpEndpoint() + "/files/duplicates",
options: Options(
headers: {
"X-Auth-Token": Configuration.instance.getToken(),
},
),
);
final dupes = DuplicateFilesResponse.fromMap(response.data);
final ids = <int>[];
for (final dupe in dupes.duplicates) {
ids.addAll(dupe.fileIDs);
}
final fileMap = await FilesDB.instance.getFilesFromIDs(ids);
final result = <DuplicateFiles>[];
for (final dupe in dupes.duplicates) {
final files = <File>[];
final Map<int, int> creationTimeCounter = {};
int mostFrequentCreationTime = 0, mostFrequentCreationTimeCount = 0;
for (final id in dupe.fileIDs) {
final file = fileMap[id];
if (file != null) {
if (creationTimeCounter.containsKey(file.creationTime)) {
creationTimeCounter[file.creationTime]++;
} else {
creationTimeCounter[file.creationTime] = 0;
}
if (creationTimeCounter[file.creationTime] >
mostFrequentCreationTimeCount) {
mostFrequentCreationTimeCount =
creationTimeCounter[file.creationTime];
mostFrequentCreationTime = file.creationTime;
}
files.add(file);
} else {
_logger.severe(
"Could not find file in local DB",
InvalidStateError(
"Could not find file in local DB " + id.toString()));
}
}
final incorrectDuplicates = <File>{};
for (final file in files) {
if (file.creationTime != mostFrequentCreationTime) {
incorrectDuplicates.add(file);
}
}
files.removeWhere((file) => incorrectDuplicates.contains(file));
if (files.length > 1) {
result.add(DuplicateFiles(files, dupe.size));
}
}
result.sort((a, b) {
final aSize = a.files.length * a.size;
final bSize = b.files.length * b.size;
return bSize - aSize;
});
return result;
} catch (e) {
_logger.severe(e);
rethrow;
}
}
Future<BackupStatus> getBackupStatus() async {
final ids = await FilesDB.instance.getBackedUpIDs();
final size = await _getFileSize(ids.uploadedIDs);

4
lib/ui/settings/backup_section_widget.dart
View file

@ -4,6 +4,7 @@ import 'package:flutter/material.dart';
import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/models/backup_status.dart';
import 'package:photos/services/deduplication_service.dart';
import 'package:photos/services/sync_service.dart';
import 'package:photos/ui/backup_folder_selection_page.dart';
import 'package:photos/ui/deduplicate_page.dart';
@ -132,7 +133,8 @@ class BackupSectionWidgetState extends State<BackupSectionWidget> {
onTap: () async {
final dialog = createProgressDialog(context, "calculating...");
await dialog.show();
final duplicates = await SyncService.instance.getDuplicateFiles();
final duplicates =
await DeduplicationService.instance.getDuplicateFiles();
await dialog.hide();
if (duplicates.isEmpty) {
showErrorDialog(context, "✨ no duplicates",