0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[mobile][photos] Reupload files with missing GPS data (#1263)

Browse source 
## Description

- Fixes corrupt files (missing GPS data) that were uploaded due to [this
issue](https://github.com/ente-io/ente/pull/1261)
- Refactor

## Tests

Tested and working
- Uploaded two file from a build that has missing permission for
`ACCESS_MEDIA_LOCATION` and GPS data is missing.
- Created a new build with changes in this PR.
- Deleted the file from device. 
- Remote file has GPS data when checked from file info.

---------

Co-authored-by: Neeraj Gupta <254676+ua741@users.noreply.github.com>
This commit is contained in:
Ashil 2024-04-01 16:41:33 +05:30 • committed by GitHub
parent 881c94be05
commit f8febe12df
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 172 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
mobile/lib/db/file_updation_db.dart
View file

@ -15,6 +15,7 @@ class FileUpdationDB {
static const columnLocalID = 'local_id';
static const columnReason = 'reason';
static const livePhotoCheck = 'livePhotoCheck';
static const androidMissingGPS = 'androidMissingGPS';
static const modificationTimeUpdated = 'modificationTimeUpdated';

18
mobile/lib/db/files_db.dart
View file

@ -1533,6 +1533,24 @@ class FilesDB {
return result;
}
Future<List<String>> getLocalFilesBackedUpWithoutLocation(int userId) async {
final db = await instance.database;
final rows = await db.query(
filesTable,
columns: [columnLocalID],
distinct: true,
where:
'$columnOwnerID = ? AND $columnLocalID IS NOT NULL AND ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) '
'AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR $columnLongitude = 0.0 or $columnLongitude = 0.0)',
whereArgs: [userId],
);
final result = <String>[];
for (final row in rows) {
result.add(row[columnLocalID] as String);
}
return result;
}
// updateSizeForUploadIDs takes a map of upploadedFileID and fileSize and
// update the fileSize for the given uploadedFileID
Future<void> updateSizeForUploadIDs(

133
mobile/lib/services/local_file_update_service.dart
View file

@ -3,6 +3,7 @@ import 'dart:core';
import 'dart:io';
import 'package:logging/logging.dart';
import "package:photo_manager/photo_manager.dart";
import "package:photos/core/configuration.dart";
import 'package:photos/core/errors.dart';
import 'package:photos/db/file_updation_db.dart';
@ -25,6 +26,10 @@ class LocalFileUpdateService {
late Logger _logger;
final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_check';
final String _doneLivePhotoImport = 'fm_import_ios_live_photo_check';
final String _androidMissingGPSImportDone =
'fm_android_missing_gps_import_done';
final String _androidMissingGPSCheckDone =
'fm_android_missing_gps_check_done';
static int twoHundredKb = 200 * 1024;
final List<String> _oldMigrationKeys = [
'fm_badCreationTime',
@ -63,6 +68,9 @@ class LocalFileUpdateService {
if (!Platform.isAndroid) {
await _handleLivePhotosSizedCheck();
}
if (Platform.isAndroid) {
await _androidMissingGPSCheck();
}
} catch (e, s) {
_logger.severe('failed to perform migration', e, s);
} finally {
@ -385,6 +393,131 @@ class LocalFileUpdateService {
await _prefs.setBool(_doneLivePhotoImport, true);
}
//#region Android Missing GPS specific methods ###
Future<void> _androidMissingGPSCheck() async {
if (_prefs.containsKey(_androidMissingGPSCheckDone)) {
return;
}
await _importAndroidBadGPSCandidate();
// singleRunLimit indicates number of files to check during single
// invocation of this method. The limit act as a crude way to limit the
// resource consumed by the method
const int singleRunLimit = 500;
final localIDsToProcess =
await _fileUpdationDB.getLocalIDsForPotentialReUpload(
singleRunLimit,
FileUpdationDB.androidMissingGPS,
);
if (localIDsToProcess.isNotEmpty) {
final chunksOf50 = localIDsToProcess.chunks(50);
for (final chunk in chunksOf50) {
final sTime = DateTime.now().microsecondsSinceEpoch;
final List<Future> futures = [];
final chunkOf10 = chunk.chunks(10);
for (final smallChunk in chunkOf10) {
futures.add(_checkForMissingGPS(smallChunk));
}
await Future.wait(futures);
final eTime = DateTime.now().microsecondsSinceEpoch;
final d = Duration(microseconds: eTime - sTime);
_logger.info(
'Performed missing GPS Location check for ${chunk.length} files '
'completed in ${d.inSeconds.toString()} secs',
);
}
} else {
_logger.info('Completed android missing GPS check');
await _prefs.setBool(_androidMissingGPSCheckDone, true);
}
}
Future<void> _checkForMissingGPS(List<String> localIDs) async {
try {
final List<EnteFile> localFiles =
await FilesDB.instance.getLocalFiles(localIDs);
final ownerID = Configuration.instance.getUserID()!;
final Set<String> localIDsWithFile = {};
final Set<String> reuploadCandidate = {};
final Set<String> processedIDs = {};
for (EnteFile file in localFiles) {
if (file.localID == null) continue;
// ignore files that are not uploaded or have different owner
if (!file.isUploaded || file.ownerID! != ownerID) {
processedIDs.add(file.localID!);
}
if (file.hasLocation) {
processedIDs.add(file.localID!);
}
}
for (EnteFile enteFile in localFiles) {
try {
if (enteFile.localID == null ||
processedIDs.contains(enteFile.localID!)) {
continue;
}
final localID = enteFile.localID!;
localIDsWithFile.add(localID);
final AssetEntity? entity = await AssetEntity.fromId(localID);
if (entity == null) {
processedIDs.add(localID);
} else {
final latLng = await entity.latlngAsync();
if ((latLng.longitude ?? 0) == 0 || (latLng.latitude ?? 0) == 0) {
processedIDs.add(localID);
} else {
reuploadCandidate.add(localID);
processedIDs.add(localID);
}
}
} catch (e, s) {
processedIDs.add(enteFile.localID!);
_logger.severe('lat/long check file ${enteFile.toString()}', e, s);
}
}
for (String id in localIDs) {
// if the file with given localID doesn't exist, consider it as done.
if (!localIDsWithFile.contains(id)) {
processedIDs.add(id);
}
}
await FileUpdationDB.instance.insertMultiple(
reuploadCandidate.toList(),
FileUpdationDB.modificationTimeUpdated,
);
await FileUpdationDB.instance.deleteByLocalIDs(
processedIDs.toList(),
FileUpdationDB.androidMissingGPS,
);
} catch (e, s) {
_logger.severe('error while checking missing GPS', e, s);
}
}
Future<void> _importAndroidBadGPSCandidate() async {
if (_prefs.containsKey(_androidMissingGPSImportDone)) {
return;
}
final sTime = DateTime.now().microsecondsSinceEpoch;
_logger.info('importing files without missing GPS');
final int ownerID = Configuration.instance.getUserID()!;
final fileLocalIDs =
await FilesDB.instance.getLocalFilesBackedUpWithoutLocation(ownerID);
await _fileUpdationDB.insertMultiple(
fileLocalIDs,
FileUpdationDB.androidMissingGPS,
);
final eTime = DateTime.now().microsecondsSinceEpoch;
final d = Duration(microseconds: eTime - sTime);
_logger.info(
'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds',
);
await _prefs.setBool(_androidMissingGPSImportDone, true);
}
//#endregion Android Missing GPS specific methods ###
Future<MediaUploadData> getUploadData(EnteFile file) async {
final mediaUploadData = await getUploadDataFromEnteFile(file);
// delete the file from app's internal cache if it was copied to app

10
mobile/lib/services/local_sync_service.dart
View file

@ -20,6 +20,7 @@ import 'package:photos/services/app_lifecycle_service.dart';
import "package:photos/services/ignored_files_service.dart";
import 'package:photos/services/local/local_sync_util.dart';
import "package:photos/utils/debouncer.dart";
import "package:photos/utils/photo_manager_util.dart";
import 'package:shared_preferences/shared_preferences.dart';
import 'package:sqflite/sqflite.dart';
import 'package:tuple/tuple.dart';
@ -61,14 +62,7 @@ class LocalSyncService {
return;
}
if (Platform.isAndroid && AppLifecycleService.instance.isForeground) {
final permissionState = await PhotoManager.requestPermissionExtend(
requestOption: const PermissionRequestOption(
androidPermission: AndroidPermission(
type: RequestType.common,
mediaLocation: true,
),
),
);
final permissionState = await requestPhotoMangerPermissions();
if (permissionState != PermissionState.authorized) {
_logger.severe(
"sync requested with invalid permission",

10
mobile/lib/ui/components/home_header_widget.dart
View file

@ -10,6 +10,7 @@ import 'package:photos/ui/components/buttons/icon_button_widget.dart';
import "package:photos/ui/settings/backup/backup_folder_selection_page.dart";
import "package:photos/utils/dialog_util.dart";
import "package:photos/utils/navigation_util.dart";
import "package:photos/utils/photo_manager_util.dart";
class HomeHeaderWidget extends StatefulWidget {
final Widget centerWidget;
@ -48,14 +49,7 @@ class _HomeHeaderWidgetState extends State<HomeHeaderWidget> {
onTap: () async {
try {
final PermissionState state =
await PhotoManager.requestPermissionExtend(
requestOption: const PermissionRequestOption(
androidPermission: AndroidPermission(
type: RequestType.common,
mediaLocation: true,
),
),
);
await requestPhotoMangerPermissions();
await LocalSyncService.instance.onUpdatePermission(state);
} on Exception catch (e) {
Logger("HomeHeaderWidget").severe(

10
mobile/lib/ui/home/grant_permissions_widget.dart
View file

@ -4,6 +4,7 @@ import 'package:flutter/material.dart';
import 'package:photo_manager/photo_manager.dart';
import "package:photos/generated/l10n.dart";
import 'package:photos/services/sync_service.dart';
import "package:photos/utils/photo_manager_util.dart";
import "package:styled_text/styled_text.dart";
class GrantPermissionsWidget extends StatelessWidget {
@ -91,14 +92,7 @@ class GrantPermissionsWidget extends StatelessWidget {
key: const ValueKey("grantPermissionButton"),
child: Text(S.of(context).grantPermission),
onPressed: () async {
final state = await PhotoManager.requestPermissionExtend(
requestOption: const PermissionRequestOption(
androidPermission: AndroidPermission(
type: RequestType.common,
mediaLocation: true,
),
),
);
final state = await requestPhotoMangerPermissions();
if (state == PermissionState.authorized ||
state == PermissionState.limited) {
await SyncService.instance.onPermissionGranted(state);

10
mobile/lib/ui/viewer/gallery/hooks/add_photos_sheet.dart
View file

@ -21,6 +21,7 @@ import "package:photos/ui/components/models/button_type.dart";
import "package:photos/ui/components/title_bar_title_widget.dart";
import "package:photos/ui/viewer/gallery/gallery.dart";
import "package:photos/utils/dialog_util.dart";
import "package:photos/utils/photo_manager_util.dart";
import 'package:wechat_assets_picker/wechat_assets_picker.dart';
Future<dynamic> showAddPhotosSheet(
@ -203,14 +204,7 @@ class AddPhotosPhotoWidget extends StatelessWidget {
}
} catch (e) {
if (e is StateError) {
final PermissionState ps = await PhotoManager.requestPermissionExtend(
requestOption: const PermissionRequestOption(
androidPermission: AndroidPermission(
type: RequestType.common,
mediaLocation: true,
),
),
);
final PermissionState ps = await requestPhotoMangerPermissions();
if (ps != PermissionState.authorized && ps != PermissionState.limited) {
await showChoiceDialog(
context,

12
mobile/lib/utils/photo_manager_util.dart Normal file
View file

@ -0,0 +1,12 @@
import "package:photo_manager/photo_manager.dart";
Future<PermissionState> requestPhotoMangerPermissions() {
return PhotoManager.requestPermissionExtend(
requestOption: const PermissionRequestOption(
androidPermission: AndroidPermission(
type: RequestType.common,
mediaLocation: true,
),
),
);
}