Merge branch 'master' into redesign-settings-new

.github/workflows/release.yml

@@ -49,14 +49,22 @@ jobs:
           SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }}
           SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }}
 
+      - name: Checksum
+        run: sha256sum build/app/outputs/flutter-apk/ente.apk > build/app/outputs/flutter-apk/checksum
+
       # Upload generated apk to the artifacts.
       - uses: actions/upload-artifact@v2
         with:
           name: release-apk
           path: build/app/outputs/flutter-apk/ente.apk
 
+      - uses: actions/upload-artifact@v2
+        with:
+          name: release-checksum
+          path: build/app/outputs/flutter-apk/checksum
+
       # Create a Github release
       - uses: ncipollo/release-action@v1
         with:
-          artifacts: "build/app/outputs/flutter-apk/ente.apk"
+          artifacts: "build/app/outputs/flutter-apk/ente.apk,build/app/outputs/flutter-apk/checksum"
           token: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -17,6 +17,7 @@ This repository contains the code for our mobile apps, built with a lot of ❤
 - Family plans
 - Shareable links for albums
 - Highlights of memories from previous years
+- Search by album, day, month, year, and file types (more in the works...)
 - Ability to detect and delete duplicate files
 - Light and dark mode
 - Image editor

lib/db/device_files_db.dart

@@ -357,20 +357,44 @@ extension DeviceFiles on FilesDB {
             orElse: () => null,
           );
           if (deviceCollection.thumbnail == null) {
-            //todo: find another image which is already imported in db for
-            // this collection
-            _logger.warning(
-              'Failed to find coverThumbnail for ${deviceCollection.name}',
-            );
-            continue;
+            final File result =
+                await getDeviceCollectionThumbnail(deviceCollection.id);
+            if (result == null) {
+              _logger.severe(
+                'Failed to find coverThumbnail for deviceFolder',
+              );
+              continue;
+            } else {
+              deviceCollection.thumbnail = result;
+            }
           }
         }
         deviceCollections.add(deviceCollection);
       }
+      if (includeCoverThumbnail) {
+        deviceCollections.sort((a, b) =>
+            b.thumbnail.creationTime.compareTo(a.thumbnail.creationTime));
+      }
       return deviceCollections;
     } catch (e) {
       _logger.severe('Failed to getDeviceCollections', e);
       rethrow;
     }
   }
+
+  Future<File> getDeviceCollectionThumbnail(String pathID) async {
+    debugPrint("Call fallback method to get potential thumbnail");
+    final db = await database;
+    final fileRows = await db.rawQuery(
+      '''SELECT * FROM FILES  f JOIN device_files df on f.local_id = df.id 
+      and df.path_id=$pathID order by f.modification_time DESC limit 1;
+          ''',
+    );
+    final files = convertToFiles(fileRows);
+    if (files.isNotEmpty) {
+      return files.first;
+    } else {
+      return null;
+    }
+  }
 }

lib/db/files_db.dart

@@ -81,6 +81,7 @@ class FilesDB {
     ...addPubMagicMetadataColumns(),
     ...createOnDeviceFilesAndPathCollection(),
     ...addFileSizeColumn(),
+    ...updateIndexes(),
   ];
 
   final dbConfig = MigrationConfig(
@@ -341,6 +342,17 @@ class FilesDB {
     ];
   }
 
+  static List<String> updateIndexes() {
+    return [
+      '''
+      DROP INDEX IF EXISTS device_folder_index;
+      ''',
+      '''
+      CREATE INDEX IF NOT EXISTS file_hash_index ON $filesTable($columnHash);
+      ''',
+    ];
+  }
+
   Future<void> clearTable() async {
     final db = await instance.database;
     await db.delete(filesTable);
@@ -879,28 +891,6 @@ class FilesDB {
     return convertToFiles(rows);
   }
 
-  Future<List<File>> getMatchingFiles(
-    String localID,
-    FileType fileType,
-    String title,
-    String deviceFolder,
-  ) async {
-    final db = await instance.database;
-    final rows = await db.query(
-      filesTable,
-      where: '''$columnTitle=? AND $columnDeviceFolder=?''',
-      whereArgs: [
-        title,
-        deviceFolder,
-      ],
-    );
-    if (rows.isNotEmpty) {
-      return convertToFiles(rows);
-    } else {
-      return null;
-    }
-  }
-
   Future<List<File>> getUploadedFilesWithHashes(
     FileHashData hashData,
     FileType fileType,
@@ -1151,37 +1141,6 @@ class FilesDB {
     return result;
   }
 
-  Future<List<File>> getLatestLocalFiles() async {
-    final db = await instance.database;
-    final rows = await db.rawQuery(
-      '''
-      SELECT $filesTable.*
-      FROM $filesTable
-      INNER JOIN
-        (
-          SELECT $columnDeviceFolder, MAX($columnCreationTime) AS max_creation_time
-          FROM $filesTable
-          WHERE $filesTable.$columnLocalID IS NOT NULL
-          GROUP BY $columnDeviceFolder
-        ) latest_files
-        ON $filesTable.$columnDeviceFolder = latest_files.$columnDeviceFolder
-        AND $filesTable.$columnCreationTime = latest_files.max_creation_time;
-    ''',
-    );
-    final files = convertToFiles(rows);
-    // TODO: Do this de-duplication within the SQL Query
-    final folderMap = <String, File>{};
-    for (final file in files) {
-      if (folderMap.containsKey(file.deviceFolder)) {
-        if (folderMap[file.deviceFolder].updationTime < file.updationTime) {
-          continue;
-        }
-      }
-      folderMap[file.deviceFolder] = file;
-    }
-    return folderMap.values.toList();
-  }
-
   Future<List<File>> getLatestCollectionFiles() async {
     debugPrint("Fetching latestCollectionFiles from db");
     String query;
@@ -1235,40 +1194,6 @@ class FilesDB {
     return collectionMap.values.toList();
   }
 
-  Future<Map<String, int>> getFileCountInDeviceFolders() async {
-    final db = await instance.database;
-    final rows = await db.rawQuery(
-      '''
-      SELECT COUNT(DISTINCT($columnLocalID)) as count, $columnDeviceFolder
-      FROM $filesTable
-      WHERE $columnLocalID IS NOT NULL
-      GROUP BY $columnDeviceFolder
-    ''',
-    );
-    final result = <String, int>{};
-    for (final row in rows) {
-      result[row[columnDeviceFolder]] = row["count"];
-    }
-    return result;
-  }
-
-  Future<List<String>> getLocalFilesBackedUpWithoutLocation() async {
-    final db = await instance.database;
-    final rows = await db.query(
-      filesTable,
-      columns: [columnLocalID],
-      distinct: true,
-      where:
-          '$columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) '
-          'AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)',
-    );
-    final result = <String>[];
-    for (final row in rows) {
-      result.add(row[columnLocalID]);
-    }
-    return result;
-  }
-
   Future<void> markForReUploadIfLocationMissing(List<String> localIDs) async {
     if (localIDs.isEmpty) {
       return;

lib/services/local/local_sync_util.dart

@@ -193,6 +193,10 @@ Future<List<File>> _convertLocalAssetsToUniqueFiles(
     for (final String localID in localPathAsset.localIDs) {
       if (!alreadySeenLocalIDs.contains(localID)) {
         final assetEntity = await AssetEntity.fromId(localID);
+        if (assetEntity == null) {
+          _logger.warning('Failed to fetch asset with id $localID');
+          continue;
+        }
         files.add(
           await File.fromAsset(localPathName, assetEntity),
         );
@@ -230,8 +234,8 @@ Future<List<AssetPathEntity>> _getGalleryList({
 
   if (updateFromTime != null && updateToTime != null) {
     filterOptionGroup.updateTimeCond = DateTimeCond(
-      min: DateTime.fromMicrosecondsSinceEpoch(updateFromTime),
-      max: DateTime.fromMicrosecondsSinceEpoch(updateToTime),
+      min: DateTime.fromMillisecondsSinceEpoch(updateFromTime ~/ 1000),
+      max: DateTime.fromMillisecondsSinceEpoch(updateToTime ~/ 1000),
     );
   }
   filterOptionGroup.containsPathModified = containsModifiedPath;
@@ -283,10 +287,10 @@ Future<Tuple2<Set<String>, List<File>>> _getLocalIDsAndFilesFromAssets(
   for (AssetEntity entity in assetList) {
     localIDs.add(entity.id);
     final bool assetCreatedOrUpdatedAfterGivenTime = max(
-          entity.createDateTime.microsecondsSinceEpoch,
-          entity.modifiedDateTime.microsecondsSinceEpoch,
-        ) >
-        fromTime;
+          entity.createDateTime.millisecondsSinceEpoch,
+          entity.modifiedDateTime.millisecondsSinceEpoch,
+        ) >=
+        (fromTime / ~1000);
     if (!alreadySeenLocalIDs.contains(entity.id) &&
         assetCreatedOrUpdatedAfterGivenTime) {
       try {

lib/services/local_file_update_service.dart

@@ -6,7 +6,6 @@ import 'dart:io';
 
 import 'package:flutter/foundation.dart';
 import 'package:logging/logging.dart';
-import 'package:photo_manager/photo_manager.dart';
 import 'package:photos/db/file_updation_db.dart';
 import 'package:photos/db/files_db.dart';
 import 'package:photos/models/file.dart' as ente;
@@ -16,7 +15,6 @@ import 'package:shared_preferences/shared_preferences.dart';
 // LocalFileUpdateService tracks all the potential local file IDs which have
 // changed/modified on the device and needed to be uploaded again.
 class LocalFileUpdateService {
-  FilesDB _filesDB;
   FileUpdationDB _fileUpdationDB;
   SharedPreferences _prefs;
   Logger _logger;
@@ -26,7 +24,6 @@ class LocalFileUpdateService {
 
   LocalFileUpdateService._privateConstructor() {
     _logger = Logger((LocalFileUpdateService).toString());
-    _filesDB = FilesDB.instance;
     _fileUpdationDB = FileUpdationDB.instance;
   }
 
@@ -37,11 +34,6 @@ class LocalFileUpdateService {
   static LocalFileUpdateService instance =
       LocalFileUpdateService._privateConstructor();
 
-  Future<bool> _markLocationMigrationAsCompleted() async {
-    _logger.info('marking migration as completed');
-    return _prefs.setBool(isLocationMigrationComplete, true);
-  }
-
   bool isLocationMigrationCompleted() {
     return _prefs.get(isLocationMigrationComplete) ?? false;
   }
@@ -53,10 +45,6 @@ class LocalFileUpdateService {
     }
     _existingMigration = Completer<void>();
     try {
-      if (!isLocationMigrationCompleted() && Platform.isAndroid) {
-        _logger.info("start migration for missing location");
-        await _runMigrationForFilesWithMissingLocation();
-      }
       await _markFilesWhichAreActuallyUpdated();
     } catch (e, s) {
       _logger.severe('failed to perform migration', e, s);
@@ -72,27 +60,26 @@ class LocalFileUpdateService {
   // then it marks the file for file update.
   Future<void> _markFilesWhichAreActuallyUpdated() async {
     final sTime = DateTime.now().microsecondsSinceEpoch;
-    bool hasData = true;
-    const int limitInBatch = 100;
-    while (hasData) {
-      final localIDsToProcess =
-          await _fileUpdationDB.getLocalIDsForPotentialReUpload(
-        limitInBatch,
-        FileUpdationDB.modificationTimeUpdated,
+    // singleRunLimit indicates number of files to check during single
+    // invocation of this method. The limit act as a crude way to limit the
+    // resource consumed by the method
+    const int singleRunLimit = 10;
+    final localIDsToProcess =
+        await _fileUpdationDB.getLocalIDsForPotentialReUpload(
+      singleRunLimit,
+      FileUpdationDB.modificationTimeUpdated,
+    );
+    if (localIDsToProcess.isNotEmpty) {
+      await _checkAndMarkFilesWithDifferentHashForFileUpdate(
+        localIDsToProcess,
+      );
+      final eTime = DateTime.now().microsecondsSinceEpoch;
+      final d = Duration(microseconds: eTime - sTime);
+      _logger.info(
+        'Performed hashCheck for ${localIDsToProcess.length} updated files '
+        'completed in ${d.inSeconds.toString()} secs',
       );
-      if (localIDsToProcess.isEmpty) {
-        hasData = false;
-      } else {
-        await _checkAndMarkFilesWithDifferentHashForFileUpdate(
-          localIDsToProcess,
-        );
-      }
     }
-    final eTime = DateTime.now().microsecondsSinceEpoch;
-    final d = Duration(microseconds: eTime - sTime);
-    _logger.info(
-      '_markFilesWhichAreActuallyUpdated migration completed in ${d.inSeconds.toString()} seconds',
-    );
   }
 
   Future<void> _checkAndMarkFilesWithDifferentHashForFileUpdate(
@@ -152,91 +139,4 @@ class LocalFileUpdateService {
     }
     return mediaUploadData;
   }
-
-  Future<void> _runMigrationForFilesWithMissingLocation() async {
-    if (!Platform.isAndroid) {
-      return;
-    }
-    // migration only needs to run if Android API Level is 29 or higher
-    final int version = int.parse(await PhotoManager.systemVersion());
-    final bool isMigrationRequired = version >= 29;
-    if (isMigrationRequired) {
-      await _importLocalFilesForMigration();
-      final sTime = DateTime.now().microsecondsSinceEpoch;
-      bool hasData = true;
-      const int limitInBatch = 100;
-      while (hasData) {
-        final localIDsToProcess =
-            await _fileUpdationDB.getLocalIDsForPotentialReUpload(
-          limitInBatch,
-          FileUpdationDB.missingLocation,
-        );
-        if (localIDsToProcess.isEmpty) {
-          hasData = false;
-        } else {
-          await _checkAndMarkFilesWithLocationForReUpload(localIDsToProcess);
-        }
-      }
-      final eTime = DateTime.now().microsecondsSinceEpoch;
-      final d = Duration(microseconds: eTime - sTime);
-      _logger.info(
-        'filesWithMissingLocation migration completed in ${d.inSeconds.toString()} seconds',
-      );
-    }
-    await _markLocationMigrationAsCompleted();
-  }
-
-  Future<void> _checkAndMarkFilesWithLocationForReUpload(
-    List<String> localIDsToProcess,
-  ) async {
-    _logger.info("files to process ${localIDsToProcess.length}");
-    final localIDsWithLocation = <String>[];
-    for (var localID in localIDsToProcess) {
-      bool hasLocation = false;
-      try {
-        final assetEntity = await AssetEntity.fromId(localID);
-        if (assetEntity == null) {
-          continue;
-        }
-        final latLng = await assetEntity.latlngAsync();
-        if ((latLng.longitude ?? 0.0) != 0.0 ||
-            (latLng.longitude ?? 0.0) != 0.0) {
-          _logger.finest(
-            'found lat/long ${latLng.longitude}/${latLng.longitude} for  ${assetEntity.title} ${assetEntity.relativePath} with id : $localID',
-          );
-          hasLocation = true;
-        }
-      } catch (e, s) {
-        _logger.severe('failed to get asset entity with id $localID', e, s);
-      }
-      if (hasLocation) {
-        localIDsWithLocation.add(localID);
-      }
-    }
-    _logger.info('marking ${localIDsWithLocation.length} files for re-upload');
-    await _filesDB.markForReUploadIfLocationMissing(localIDsWithLocation);
-    await _fileUpdationDB.deleteByLocalIDs(
-      localIDsToProcess,
-      FileUpdationDB.missingLocation,
-    );
-  }
-
-  Future<void> _importLocalFilesForMigration() async {
-    if (_prefs.containsKey(isLocalImportDone)) {
-      return;
-    }
-    final sTime = DateTime.now().microsecondsSinceEpoch;
-    _logger.info('importing files without location info');
-    final fileLocalIDs = await _filesDB.getLocalFilesBackedUpWithoutLocation();
-    await _fileUpdationDB.insertMultiple(
-      fileLocalIDs,
-      FileUpdationDB.missingLocation,
-    );
-    final eTime = DateTime.now().microsecondsSinceEpoch;
-    final d = Duration(microseconds: eTime - sTime);
-    _logger.info(
-      'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds',
-    );
-    await _prefs.setBool(isLocalImportDone, true);
-  }
 }

lib/services/local_sync_service.dart

@@ -331,12 +331,6 @@ class LocalSyncService {
     Set<String> editedFileIDs,
     Set<String> downloadedFileIDs,
   ) async {
-    _logger.info(
-      "Loading photos from " +
-          DateTime.fromMicrosecondsSinceEpoch(fromTime).toString() +
-          " to " +
-          DateTime.fromMicrosecondsSinceEpoch(toTime).toString(),
-    );
     final Tuple2<List<LocalPathAsset>, List<File>> result =
         await getLocalPathAssetsAndFiles(fromTime, toTime, _computer);
     await FilesDB.instance.insertLocalAssets(
@@ -344,8 +338,13 @@ class LocalSyncService {
       shouldAutoBackup: Configuration.instance.hasSelectedAllFoldersForBackup(),
     );
     final List<File> files = result.item2;
+    _logger.info(
+      "Loaded ${files.length} photos from " +
+          DateTime.fromMicrosecondsSinceEpoch(fromTime).toString() +
+          " to " +
+          DateTime.fromMicrosecondsSinceEpoch(toTime).toString(),
+    );
     if (files.isNotEmpty) {
-      _logger.info("Fetched " + files.length.toString() + " files.");
       await _trackUpdatedFiles(
         files,
         existingLocalFileIDs,

lib/services/remote_sync_service.dart

@@ -623,7 +623,16 @@ class RemoteSyncService {
                 (e) => e.modificationTime ?? 0,
               )
               .reduce(max);
-          if (maxModificationTime > remoteDiff.modificationTime) {
+
+          /* todo: In case of iOS, we will miss any asset modification in
+            between of two installation. This is done to avoid fetching assets
+            from iCloud when modification time could have changed for number of
+            reasons. To fix this, we need to identify a way to store version
+            for the adjustments or just if the asset has been modified ever.
+            https://stackoverflow.com/a/50093266/546896
+            */
+          if (maxModificationTime > remoteDiff.modificationTime &&
+              Platform.isAndroid) {
             localButUpdatedOnDevice++;
             await FileUpdationDB.instance.insertMultiple(
               [remoteDiff.localID],

lib/services/search_service.dart

@@ -37,8 +37,9 @@ class SearchService {
 
   Future<void> init() async {
     // Intention of delay is to give more CPU cycles to other tasks
-    Future.delayed(const Duration(seconds: 5), () async {
-      /* In case home screen loads before 5 seconds and user starts search,
+    // 8 is just a magic number
+    Future.delayed(const Duration(seconds: 8), () async {
+      /* In case home screen loads before 8 seconds and user starts search,
        future will not be null.So here getAllFiles won't run again in that case. */
       if (_cachedFilesFuture == null) {
         _getAllFiles();
@@ -46,8 +47,8 @@ class SearchService {
     });
 
     Bus.instance.on<LocalPhotosUpdatedEvent>().listen((event) {
+      // only invalidate, let the load happen on demand
       _cachedFilesFuture = null;
-      _getAllFiles();
     });
   }
 

pubspec.yaml

@@ -11,7 +11,7 @@ description: ente photos application
 # In iOS, build-name is used as CFBundleShortVersionString while build-number used as CFBundleVersion.
 # Read more about iOS versioning at
 # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html
-version: 0.6.39+369
+version: 0.6.40+370
 
 environment:
   sdk: '>=2.12.0 <3.0.0'