beenull/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update diff handling

Browse source
This commit is contained in:
Vishnu Mohandas 2020-03-28 19:26:06 +05:30
parent 22d3f2c9ed
commit 22ca572532
6 changed files with 149 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
lib/db/db_helper.dart
View file

@ -11,7 +11,6 @@ class DatabaseHelper {
static final table = 'photos';
static final columnId = 'photo_id';
static final columnLocalPath = 'local_path';
static final columnUrl = 'url';
static final columnHash = 'hash';
@ -42,11 +41,10 @@ class DatabaseHelper {
Future _onCreate(Database db, int version) async {
await db.execute('''
CREATE TABLE $table (
$columnId VARCHAR(255) PRIMARY KEY,
$columnLocalPath TEXT NOT NULL,
$columnUrl TEXT NOT NULL,
$columnUrl TEXT,
$columnHash TEXT NOT NULL,
$columnSyncTimestamp TEXT NOT NULL
$columnSyncTimestamp TEXT
)
''');
}
@ -54,7 +52,6 @@ class DatabaseHelper {
Future<int> insertPhoto(Photo photo) async {
Database db = await instance.database;
var row = new Map<String, dynamic>();
row[columnId] = photo.photoID;
row[columnLocalPath] = photo.localPath;
row[columnUrl] = photo.url;
row[columnHash] = photo.hash;
@ -65,49 +62,47 @@ class DatabaseHelper {
Future<List<Photo>> getAllPhotos() async {
Database db = await instance.database;
var results = await db.query(table);
return _convertToPhotos(results);
}
Future<List<Photo>> getPhotosToBeUploaded() async {
Database db = await instance.database;
var results = await db.query(table, where: '$columnUrl IS NULL');
return _convertToPhotos(results);
}
// We are assuming here that the hash column in the map is set. The other
// column values will be used to update the row.
Future<int> updateUrlAndTimestamp(
String hash, String url, String timestamp) async {
Database db = await instance.database;
var row = new Map<String, dynamic>();
row[columnUrl] = url;
row[columnSyncTimestamp] = timestamp;
return await db
.update(table, row, where: '$columnHash = ?', whereArgs: [hash]);
}
Future<bool> containsPath(String path) async {
Database db = await instance.database;
return (await db
.query(table, where: '$columnLocalPath =?', whereArgs: [path]))
.length >
0;
}
Future<bool> containsPhotoHash(String hash) async {
Database db = await instance.database;
return (await db.query(table, where: '$columnHash =?', whereArgs: [hash]))
.length >
0;
}
List<Photo> _convertToPhotos(List<Map<String, dynamic>> results) {
var photos = List<Photo>();
for (var result in results) {
photos.add(Photo.fromRow(result));
}
return photos;
}
// Helper methods
// Inserts a row in the database where each key in the Map is a column name
// and the value is the column value. The return value is the id of the
// inserted row.
Future<int> insert(Map<String, dynamic> row) async {
Database db = await instance.database;
return await db.insert(table, row);
}
// All of the rows are returned as a list of maps, where each map is
// a key-value list of columns.
Future<List<Map<String, dynamic>>> queryAllRows() async {
Database db = await instance.database;
return await db.query(table);
}
// We are assuming here that the id column in the map is set. The other
// column values will be used to update the row.
Future<int> update(Map<String, dynamic> row) async {
Database db = await instance.database;
int id = row[columnId];
return await db.update(table, row, where: '$columnId = ?', whereArgs: [id]);
}
// Deletes the row specified by the id. The number of affected rows is
// returned. This should be 1 as long as the row exists.
Future<int> delete(int id) async {
Database db = await instance.database;
return await db.delete(table, where: '$columnId = ?', whereArgs: [id]);
}
Future<bool> containsPath(String path) async {
Database db = await instance.database;
return (await db.query(table, where: '$columnLocalPath =?', whereArgs: [path]))
.length >
0;
}
}

3
lib/main.dart
View file

@ -18,7 +18,8 @@ void main() async {
WidgetsFlutterBinding.ensureInitialized();
await provider.refreshGalleryList();
var assets = await provider.list[0].assetList;
PhotoSyncManager(assets);
var photoSyncManager = PhotoSyncManager(assets);
await photoSyncManager.init();
runApp(MyApp2());
}

31
lib/models/photo.dart
View file

@ -1,19 +1,38 @@
import 'dart:io';
import 'package:crypto/crypto.dart';
import 'package:photo_manager/photo_manager.dart';
class Photo {
String photoID;
String url;
String localPath;
String hash;
int syncTimestamp;
Photo();
Photo.fromJson(Map<String, dynamic> json)
: photoID = json["photoID"],
url = json["url"],
: url = json["url"],
hash = json["hash"],
syncTimestamp = json["syncTimestamp"];
Photo.fromRow(Map<String, dynamic> row)
: photoID = row["photo_id"],
localPath = row["local_path"],
: localPath = row["local_path"],
url = row["url"],
hash = row["hash"],
syncTimestamp = int.parse(row["sync_timestamp"]);
syncTimestamp = row["sync_timestamp"] == null
? -1
: int.parse(row["sync_timestamp"]);
static Future<Photo> fromAsset(AssetEntity asset) async {
Photo photo = Photo();
var file = (await asset.originFile);
photo.localPath = file.path;
photo.hash = getHash(file);
return photo;
}
static String getHash(File file) {
return sha256.convert(file.readAsBytesSync()).toString();
}
}

140
lib/photo_sync_manager.dart
View file

@ -9,99 +9,121 @@ import 'package:dio/dio.dart';
import 'package:myapp/models/photo.dart';
class PhotoSyncManager {
final logger = Logger();
final dio = Dio();
final endpoint = "http://172.20.10.6:8080";
final user = "umbu";
static final lastSyncTimestampKey = "last_sync_timestamp_0";
final _logger = Logger();
final _dio = Dio();
final _endpoint = "http://192.168.0.106:8080";
final _user = "umbu";
final List<AssetEntity> _assets;
static final _lastSyncTimestampKey = "last_sync_timestamp_0";
static final _lastDBUpdateTimestampKey = "last_db_update_timestamp";
PhotoSyncManager(List<AssetEntity> assets) {
logger.i("PhotoSyncManager init");
_syncPhotos(assets);
PhotoSyncManager(this._assets) {
_logger.i("PhotoSyncManager init");
_assets.sort((first, second) => second
.modifiedDateTime.millisecondsSinceEpoch
.compareTo(first.modifiedDateTime.millisecondsSinceEpoch));
}
_syncPhotos(List<AssetEntity> assets) async {
Future<void> init() async {
await _updateDatabase();
await _syncPhotos();
}
Future<bool> _updateDatabase() async {
SharedPreferences prefs = await SharedPreferences.getInstance();
var lastSyncTimestamp = prefs.getInt(lastSyncTimestampKey);
var lastDBUpdateTimestamp = prefs.getInt(_lastDBUpdateTimestampKey);
if (lastDBUpdateTimestamp == null) {
lastDBUpdateTimestamp = 0;
}
for (AssetEntity asset in _assets) {
if (asset.createDateTime.millisecondsSinceEpoch > lastDBUpdateTimestamp) {
await DatabaseHelper.instance.insertPhoto(await Photo.fromAsset(asset));
}
}
return await prefs.setInt(
_lastDBUpdateTimestampKey, DateTime.now().millisecondsSinceEpoch);
}
_syncPhotos() async {
SharedPreferences prefs = await SharedPreferences.getInstance();
var lastSyncTimestamp = prefs.getInt(_lastSyncTimestampKey);
if (lastSyncTimestamp == null) {
lastSyncTimestamp = 0;
}
logger.i("Last sync timestamp: " + lastSyncTimestamp.toString());
_logger.i("Last sync timestamp: " + lastSyncTimestamp.toString());
await _downloadDiff(lastSyncTimestamp, prefs);
List<Photo> diff = await _getDiff(lastSyncTimestamp);
await _downloadDiff(diff, prefs);
await _uploadDiff(assets, prefs);
await _uploadDiff(prefs);
// TODO: Fix race conditions triggered due to concurrent syncs.
// Add device_id/last_sync_timestamp to the upload request?
}
Future _uploadDiff(List<AssetEntity> assets, SharedPreferences prefs) async {
assets.sort((first, second) => second
.modifiedDateTime.millisecondsSinceEpoch
.compareTo(first.modifiedDateTime.millisecondsSinceEpoch));
var uploadedAssetCount = 0;
for (AssetEntity asset in assets) {
Future _uploadDiff(SharedPreferences prefs) async {
var uploadedCount = 0;
List<Photo> photosToBeUploaded =
await DatabaseHelper.instance.getPhotosToBeUploaded();
for (Photo photo in photosToBeUploaded) {
// TODO: Fix me
if (uploadedAssetCount == 100) {
if (uploadedCount == 100) {
return;
}
var containsPath = await DatabaseHelper.instance
.containsPath((await asset.originFile).path);
if (!containsPath) {
var response = await _uploadFile(asset);
prefs.setInt(lastSyncTimestampKey, response.syncTimestamp);
uploadedAssetCount++;
}
var uploadedPhoto = await _uploadFile(photo.localPath, photo.hash);
await DatabaseHelper.instance.updateUrlAndTimestamp(photo.hash,
uploadedPhoto.url, uploadedPhoto.syncTimestamp.toString());
prefs.setInt(_lastSyncTimestampKey, uploadedPhoto.syncTimestamp);
uploadedCount++;
}
}
Future _downloadDiff(int lastSyncTimestamp, SharedPreferences prefs) async {
Response response = await dio.get(endpoint + "/diff", queryParameters: {
"user": user,
Future _downloadDiff(List<Photo> diff, SharedPreferences prefs) async {
var externalPath = (await getApplicationDocumentsDirectory()).path;
_logger.i("External path: " + externalPath);
var path = externalPath + "/photos/";
for (Photo photo in diff) {
if (await DatabaseHelper.instance.containsPhotoHash(photo.hash)) {
await DatabaseHelper.instance.updateUrlAndTimestamp(
photo.hash, photo.url, photo.syncTimestamp.toString());
continue;
} else {
var localPath = path + basename(photo.url);
await _dio.download(_endpoint + photo.url, localPath);
photo.localPath = localPath;
await insertPhotoToDB(photo);
}
await prefs.setInt(_lastSyncTimestampKey, photo.syncTimestamp);
}
}
Future<List<Photo>> _getDiff(int lastSyncTimestamp) async {
Response response = await _dio.get(_endpoint + "/diff", queryParameters: {
"user": _user,
"lastSyncTimestamp": lastSyncTimestamp
});
logger.i(response.toString());
var externalPath = (await getApplicationDocumentsDirectory()).path;
logger.i("External path: " + externalPath);
var path = externalPath + "/photos/";
List<Photo> photos = (response.data["diff"] as List)
_logger.i(response.toString());
return (response.data["diff"] as List)
.map((photo) => new Photo.fromJson(photo))
.toList();
for (Photo photo in photos) {
await dio.download(endpoint + photo.url, path + basename(photo.url));
photo.hash = _getHash(photo);
photo.localPath = path + basename(photo.url);
insertPhotoToDB(photo);
prefs.setInt(lastSyncTimestampKey, photo.syncTimestamp);
logger.i("Downloaded " + photo.url + " to " + path);
}
}
Future<Photo> _uploadFile(AssetEntity entity) async {
logger.i("Uploading: " + entity.id);
var path = (await entity.originFile).path;
Future<Photo> _uploadFile(String path, String hash) async {
var formData = FormData.fromMap({
"file": await MultipartFile.fromFile(path, filename: entity.title),
"user": user,
"file": await MultipartFile.fromFile(path, filename: basename(path)),
"user": _user,
});
var response = await dio.post(endpoint + "/upload", data: formData);
logger.i(response.toString());
var response = await _dio.post(_endpoint + "/upload", data: formData);
_logger.i(response.toString());
var photo = Photo.fromJson(response.data);
photo.hash = _getHash(photo);
_logger.i("Locally computed hash for " + path + ": " + hash);
_logger.i("Server computed hash for " + path + ": " + photo.hash);
photo.localPath = path;
insertPhotoToDB(photo);
return photo;
}
String _getHash(Photo photo) {
// TODO: Compute hash
return "hash";
}
Future<void> insertPhotoToDB(Photo photo) async {
logger.i("Inserting to DB");
_logger.i("Inserting to DB");
await DatabaseHelper.instance.insertPhoto(photo);
PhotoLoader.instance.reloadPhotos();
}

2
pubspec.lock
View file

@ -51,7 +51,7 @@ packages:
source: hosted
version: "2.1.1"
crypto:
dependency: transitive
dependency: "direct main"
description:
name: crypto
url: "https://pub.dartlang.org"

1
pubspec.yaml
View file

@ -31,6 +31,7 @@ dependencies:
logger: ^0.8.3
dio: ^3.0.9
local_image_provider: ^1.0.0
crypto: ^2.1.3
dev_dependencies:
flutter_test: