From 453a44cb82cc17bc47d5df1ae505379c23a7bdc3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 12:28:07 +0530 Subject: [PATCH 1/8] Update computer pkg --- lib/services/local_sync_service.dart | 2 +- lib/utils/crypto_util.dart | 4 ++-- pubspec.lock | 4 ++-- pubspec.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/services/local_sync_service.dart b/lib/services/local_sync_service.dart index eb79aeb18..cf7808417 100644 --- a/lib/services/local_sync_service.dart +++ b/lib/services/local_sync_service.dart @@ -24,7 +24,7 @@ import 'package:tuple/tuple.dart'; class LocalSyncService { final _logger = Logger("LocalSyncService"); final _db = FilesDB.instance; - final Computer _computer = Computer(); + final Computer _computer = Computer.shared(); late SharedPreferences _prefs; Completer? _existingSync; diff --git a/lib/utils/crypto_util.dart b/lib/utils/crypto_util.dart index 1a838c510..7ebb1cb4e 100644 --- a/lib/utils/crypto_util.dart +++ b/lib/utils/crypto_util.dart @@ -166,7 +166,7 @@ Uint8List chachaDecryptData(Map args) { } class CryptoUtil { - static final Computer _computer = Computer(); + static final Computer _computer = Computer.shared(); static init() { _computer.turnOn(workersCount: 4); @@ -391,7 +391,7 @@ class CryptoUtil { return DerivedKeyResult(key, memLimit, opsLimit); } - // Derives a key for a given password, salt, memLimit and opsLimit using + // Derives a key for a given password, salt, memLimit and opsLimit using // Argon2id, v1.3. static Future deriveKey( Uint8List password, diff --git a/pubspec.lock b/pubspec.lock index 71935b381..9f83b92fa 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -276,10 +276,10 @@ packages: dependency: "direct main" description: name: computer - sha256: "3df9f1ef497aaf69e066b00f4441726eb28037dc33e97b5d56393967f92c5fe8" + sha256: "34176c612a02ebf3e5eaa3e9b40e2b1df495043c46614999639d19e66823232d" url: "https://pub.dev" source: hosted - version: "2.0.0" + version: "3.2.1" confetti: dependency: "direct main" description: diff --git a/pubspec.yaml b/pubspec.yaml index 2b2e436ac..e054c00f0 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -28,7 +28,7 @@ dependencies: chewie: path: thirdparty/chewie collection: # dart - computer: ^2.0.0 + computer: ^3.2.1 confetti: ^0.6.0 connectivity_plus: ^3.0.3 crypto: ^3.0.2 From 4745a5a2f3268bcb3eaae0551e96f1899b9a40b4 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 12:33:38 +0530 Subject: [PATCH 2/8] Initiate shared workers during app init --- lib/main.dart | 3 +++ lib/services/local_sync_service.dart | 1 - lib/utils/crypto_util.dart | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/main.dart b/lib/main.dart index d08535a03..2d5da4b00 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -3,6 +3,7 @@ import 'dart:io'; import "package:adaptive_theme/adaptive_theme.dart"; import 'package:background_fetch/background_fetch.dart'; +import "package:computer/computer.dart"; import 'package:firebase_messaging/firebase_messaging.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; @@ -153,6 +154,8 @@ Future _init(bool isBackground, {String via = ''}) async { } else { AppLifecycleService.instance.onAppInForeground('init via: $via'); } + // Start workers asynchroniously. No need to wait for them to start + Computer.shared().turnOn(workersCount: 4); CryptoUtil.init(); await NotificationService.instance.init(); await NetworkClient.instance.init(); diff --git a/lib/services/local_sync_service.dart b/lib/services/local_sync_service.dart index cf7808417..cbfe7d0fb 100644 --- a/lib/services/local_sync_service.dart +++ b/lib/services/local_sync_service.dart @@ -48,7 +48,6 @@ class LocalSyncService { if (!AppLifecycleService.instance.isForeground) { await PhotoManager.setIgnorePermissionCheck(true); } - await _computer.turnOn(workersCount: 1); if (hasGrantedPermissions()) { _registerChangeCallback(); } diff --git a/lib/utils/crypto_util.dart b/lib/utils/crypto_util.dart index 7ebb1cb4e..3498af8a8 100644 --- a/lib/utils/crypto_util.dart +++ b/lib/utils/crypto_util.dart @@ -166,10 +166,10 @@ Uint8List chachaDecryptData(Map args) { } class CryptoUtil { + // Note: workers are turned on during app startup. static final Computer _computer = Computer.shared(); static init() { - _computer.turnOn(workersCount: 4); Sodium.init(); } From 54ff4e5dd55ab26088d39329bc46b8f1bce2e6c3 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 12:34:02 +0530 Subject: [PATCH 3/8] Enable computer logs in debug mode --- lib/main.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/main.dart b/lib/main.dart index 2d5da4b00..059a87885 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -155,7 +155,7 @@ Future _init(bool isBackground, {String via = ''}) async { AppLifecycleService.instance.onAppInForeground('init via: $via'); } // Start workers asynchroniously. No need to wait for them to start - Computer.shared().turnOn(workersCount: 4); + Computer.shared().turnOn(workersCount: 4, verbose: kDebugMode); CryptoUtil.init(); await NotificationService.instance.init(); await NetworkClient.instance.init(); From 85a95a52ec0fc0052b65587f07a87820d8832824 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 12:35:18 +0530 Subject: [PATCH 4/8] Fix typo --- lib/main.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/main.dart b/lib/main.dart index 059a87885..2ecb89bbb 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -154,7 +154,7 @@ Future _init(bool isBackground, {String via = ''}) async { } else { AppLifecycleService.instance.onAppInForeground('init via: $via'); } - // Start workers asynchroniously. No need to wait for them to start + // Start workers asynchronously. No need to wait for them to start Computer.shared().turnOn(workersCount: 4, verbose: kDebugMode); CryptoUtil.init(); await NotificationService.instance.init(); From 34ec149266fc66737de8a10040aa91e5f2e6ad34 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 13:26:57 +0530 Subject: [PATCH 5/8] Switch to forked version for computer --- pubspec.lock | 9 +++++---- pubspec.yaml | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pubspec.lock b/pubspec.lock index 9f83b92fa..ff9019caf 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -275,10 +275,11 @@ packages: computer: dependency: "direct main" description: - name: computer - sha256: "34176c612a02ebf3e5eaa3e9b40e2b1df495043c46614999639d19e66823232d" - url: "https://pub.dev" - source: hosted + path: "." + ref: HEAD + resolved-ref: "82e365fed8a1a76f6eea0220de98389eed7b0445" + url: "https://github.com/ente-io/computer.git" + source: git version: "3.2.1" confetti: dependency: "direct main" diff --git a/pubspec.yaml b/pubspec.yaml index e054c00f0..f3212b381 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -28,7 +28,8 @@ dependencies: chewie: path: thirdparty/chewie collection: # dart - computer: ^3.2.1 + computer: + git: "https://github.com/ente-io/computer.git" confetti: ^0.6.0 connectivity_plus: ^3.0.3 crypto: ^3.0.2 From b5fe6ede26ba3b0d5276b5b419e8ad46bd00078d Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 13:27:18 +0530 Subject: [PATCH 6/8] Assign taskname for computer tasks --- lib/services/local/local_sync_util.dart | 9 +++++++-- lib/utils/crypto_util.dart | 26 ++++++++++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/lib/services/local/local_sync_util.dart b/lib/services/local/local_sync_util.dart index 6f161517a..18b15a613 100644 --- a/lib/services/local/local_sync_util.dart +++ b/lib/services/local/local_sync_util.dart @@ -39,6 +39,8 @@ Future, List>> getLocalPathAssetsAndFiles( "alreadySeenLocalIDs": alreadySeenLocalIDs, "assetList": assetsInPath, }, + taskName: + "getLocalPathAssetsAndFiles-${pathEntity.name}-count-${assetsInPath.length}", ); alreadySeenLocalIDs.addAll(result.item1); uniqueFiles.addAll(result.item2); @@ -125,8 +127,11 @@ Future getDiffWithLocal( args['existingIDs'] = existingIDs; args['invalidIDs'] = invalidIDs; args['pathToLocalIDs'] = pathToLocalIDs; - final LocalDiffResult diffResult = - await computer.compute(_getLocalAssetsDiff, param: args); + final LocalDiffResult diffResult = await computer.compute( + _getLocalAssetsDiff, + param: args, + taskName: "getLocalAssetsDiff", + ); if (diffResult.localPathAssets != null) { diffResult.uniqueLocalFiles = await _convertLocalAssetsToUniqueFiles(diffResult.localPathAssets!); diff --git a/lib/utils/crypto_util.dart b/lib/utils/crypto_util.dart index 3498af8a8..27407b950 100644 --- a/lib/utils/crypto_util.dart +++ b/lib/utils/crypto_util.dart @@ -231,7 +231,11 @@ class CryptoUtil { args["cipher"] = cipher; args["nonce"] = nonce; args["key"] = key; - return _computer.compute(cryptoSecretboxOpenEasy, param: args); + return _computer.compute( + cryptoSecretboxOpenEasy, + param: args, + taskName: "decrypt", + ); } // Decrypts the given cipher, with the given key and nonce using XSalsa20 @@ -262,7 +266,11 @@ class CryptoUtil { final args = {}; args["source"] = source; args["key"] = key; - return _computer.compute(chachaEncryptData, param: args); + return _computer.compute( + chachaEncryptData, + param: args, + taskName: "encryptChaCha", + ); } // Decrypts the given source, with the given key and header using XChaCha20 @@ -277,7 +285,11 @@ class CryptoUtil { args["source"] = source; args["key"] = key; args["header"] = header; - return _computer.compute(chachaDecryptData, param: args); + return _computer.compute( + chachaDecryptData, + param: args, + taskName: "decryptChaCha", + ); } // Encrypts the file at sourceFilePath, with the key (if provided) and a @@ -293,7 +305,8 @@ class CryptoUtil { args["sourceFilePath"] = sourceFilePath; args["destinationFilePath"] = destinationFilePath; args["key"] = key; - return _computer.compute(chachaEncryptFile, param: args); + return _computer.compute(chachaEncryptFile, + param: args, taskName: "encryptFile"); } // Decrypts the file at sourceFilePath, with the given key and header using @@ -309,7 +322,8 @@ class CryptoUtil { args["destinationFilePath"] = destinationFilePath; args["header"] = header; args["key"] = key; - return _computer.compute(chachaDecryptFile, param: args); + return _computer.compute(chachaDecryptFile, + param: args, taskName: "decryptFile"); } // Generates and returns a 256-bit key. @@ -407,6 +421,7 @@ class CryptoUtil { "memLimit": memLimit, "opsLimit": opsLimit, }, + taskName: "deriveKey", ); } @@ -417,6 +432,7 @@ class CryptoUtil { param: { "sourceFilePath": source.path, }, + taskName: "fileHash", ); } } From 0fcf40b12aa97398193eafae4f3d8d172743ef7f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 13:55:31 +0530 Subject: [PATCH 7/8] Refactor: Directly ref shared computer instead of passing as variable --- lib/services/local/local_sync_util.dart | 7 +++---- lib/services/local_sync_service.dart | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/services/local/local_sync_util.dart b/lib/services/local/local_sync_util.dart index 18b15a613..2216e3984 100644 --- a/lib/services/local/local_sync_util.dart +++ b/lib/services/local/local_sync_util.dart @@ -16,7 +16,6 @@ const assetFetchPageSize = 2000; Future, List>> getLocalPathAssetsAndFiles( int fromTime, int toTime, - Computer computer, ) async { final pathEntities = await _getGalleryList( updateFromTime: fromTime, @@ -31,7 +30,8 @@ Future, List>> getLocalPathAssetsAndFiles( final List uniqueFiles = []; for (AssetPathEntity pathEntity in pathEntities) { final List assetsInPath = await _getAllAssetLists(pathEntity); - final Tuple2, List> result = await computer.compute( + final Tuple2, List> result = + await Computer.shared().compute( _getLocalIDsAndFilesFromAssets, param: { "pathEntity": pathEntity, @@ -120,14 +120,13 @@ Future getDiffWithLocal( Set existingIDs, // localIDs of files already imported in app Map> pathToLocalIDs, Set invalidIDs, - Computer computer, ) async { final Map args = {}; args['assets'] = assets; args['existingIDs'] = existingIDs; args['invalidIDs'] = invalidIDs; args['pathToLocalIDs'] = pathToLocalIDs; - final LocalDiffResult diffResult = await computer.compute( + final LocalDiffResult diffResult = await Computer.shared().compute( _getLocalAssetsDiff, param: args, taskName: "getLocalAssetsDiff", diff --git a/lib/services/local_sync_service.dart b/lib/services/local_sync_service.dart index cbfe7d0fb..cfd144b7a 100644 --- a/lib/services/local_sync_service.dart +++ b/lib/services/local_sync_service.dart @@ -24,7 +24,6 @@ import 'package:tuple/tuple.dart'; class LocalSyncService { final _logger = Logger("LocalSyncService"); final _db = FilesDB.instance; - final Computer _computer = Computer.shared(); late SharedPreferences _prefs; Completer? _existingSync; @@ -299,7 +298,7 @@ class LocalSyncService { required int toTime, }) async { final Tuple2, List> result = - await getLocalPathAssetsAndFiles(fromTime, toTime, _computer); + await getLocalPathAssetsAndFiles(fromTime, toTime); // Update the mapping for device path_id to local file id. Also, keep track // of newly discovered device paths From 87333c83d58620d1ebbc9179e17061bbc63192de Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 29 Apr 2023 14:03:44 +0530 Subject: [PATCH 8/8] [Perf] Avoid redundant method call via compute --- lib/services/local/local_sync_util.dart | 32 ++++++++++++++----------- lib/services/local_sync_service.dart | 17 +++++++------ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/lib/services/local/local_sync_util.dart b/lib/services/local/local_sync_util.dart index 2216e3984..9589d2ca4 100644 --- a/lib/services/local/local_sync_util.dart +++ b/lib/services/local/local_sync_util.dart @@ -30,20 +30,24 @@ Future, List>> getLocalPathAssetsAndFiles( final List uniqueFiles = []; for (AssetPathEntity pathEntity in pathEntities) { final List assetsInPath = await _getAllAssetLists(pathEntity); - final Tuple2, List> result = - await Computer.shared().compute( - _getLocalIDsAndFilesFromAssets, - param: { - "pathEntity": pathEntity, - "fromTime": fromTime, - "alreadySeenLocalIDs": alreadySeenLocalIDs, - "assetList": assetsInPath, - }, - taskName: - "getLocalPathAssetsAndFiles-${pathEntity.name}-count-${assetsInPath.length}", - ); - alreadySeenLocalIDs.addAll(result.item1); - uniqueFiles.addAll(result.item2); + late Tuple2, List> result; + if (assetsInPath.isEmpty) { + result = const Tuple2({}, []); + } else { + result = await Computer.shared().compute( + _getLocalIDsAndFilesFromAssets, + param: { + "pathEntity": pathEntity, + "fromTime": fromTime, + "alreadySeenLocalIDs": alreadySeenLocalIDs, + "assetList": assetsInPath, + }, + taskName: + "getLocalPathAssetsAndFiles-${pathEntity.name}-count-${assetsInPath.length}", + ); + alreadySeenLocalIDs.addAll(result.item1); + uniqueFiles.addAll(result.item2); + } localPathAssets.add( LocalPathAsset( localIDs: result.item1, diff --git a/lib/services/local_sync_service.dart b/lib/services/local_sync_service.dart index cfd144b7a..300592ea8 100644 --- a/lib/services/local_sync_service.dart +++ b/lib/services/local_sync_service.dart @@ -1,7 +1,6 @@ import 'dart:async'; import 'dart:io'; -import 'package:computer/computer.dart'; import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; import 'package:photo_manager/photo_manager.dart'; @@ -193,7 +192,6 @@ class LocalSyncService { existingLocalFileIDs, pathToLocalIDs, invalidIDs, - _computer, ); bool hasAnyMappingChanged = false; if (localDiffResult.newPathToLocalIDs?.isNotEmpty ?? false) { @@ -300,15 +298,16 @@ class LocalSyncService { final Tuple2, List> result = await getLocalPathAssetsAndFiles(fromTime, toTime); - // Update the mapping for device path_id to local file id. Also, keep track - // of newly discovered device paths - await FilesDB.instance.insertLocalAssets( - result.item1, - shouldAutoBackup: Configuration.instance.hasSelectedAllFoldersForBackup(), - ); - final List files = result.item2; if (files.isNotEmpty) { + // Update the mapping for device path_id to local file id. Also, keep track + // of newly discovered device paths + await FilesDB.instance.insertLocalAssets( + result.item1, + shouldAutoBackup: + Configuration.instance.hasSelectedAllFoldersForBackup(), + ); + _logger.info( "Loaded ${files.length} photos from " + DateTime.fromMicrosecondsSinceEpoch(fromTime).toString() +