file_util.dart 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. import 'dart:async';
  2. import 'dart:collection';
  3. import 'dart:io' as io;
  4. import 'dart:io';
  5. import 'dart:typed_data';
  6. import 'package:flutter/widgets.dart';
  7. import 'package:flutter_sodium/flutter_sodium.dart';
  8. import 'package:logging/logging.dart';
  9. import 'package:path/path.dart';
  10. import 'package:dio/dio.dart';
  11. import 'package:flutter_cache_manager/flutter_cache_manager.dart';
  12. import 'package:flutter_image_compress/flutter_image_compress.dart';
  13. import 'package:photo_manager/photo_manager.dart';
  14. import 'package:photos/core/cache/image_cache.dart';
  15. import 'package:photos/core/cache/thumbnail_cache.dart';
  16. import 'package:photos/core/cache/thumbnail_cache_manager.dart';
  17. import 'package:photos/core/cache/video_cache_manager.dart';
  18. import 'package:photos/core/configuration.dart';
  19. import 'package:photos/core/constants.dart';
  20. import 'package:photos/core/event_bus.dart';
  21. import 'package:photos/core/network.dart';
  22. import 'package:photos/db/files_db.dart';
  23. import 'package:photos/events/collection_updated_event.dart';
  24. import 'package:photos/models/file.dart';
  25. import 'package:photos/models/file_type.dart';
  26. import 'package:photos/repositories/file_repository.dart';
  27. import 'package:photos/services/collections_service.dart';
  28. import 'package:photos/services/sync_service.dart';
  29. import 'package:photos/utils/dialog_util.dart';
  30. import 'crypto_util.dart';
  31. final _logger = Logger("FileUtil");
  32. Future<void> deleteFilesFromEverywhere(
  33. BuildContext context, List<File> files) async {
  34. final dialog = createProgressDialog(context, "deleting...");
  35. await dialog.show();
  36. final localIDs = List<String>();
  37. for (final file in files) {
  38. if (file.localID != null) {
  39. localIDs.add(file.localID);
  40. }
  41. }
  42. final deletedIDs =
  43. (await PhotoManager.editor.deleteWithIds(localIDs)).toSet();
  44. bool hasUploadedFiles = false;
  45. for (final file in files) {
  46. if (file.localID != null) {
  47. // Remove only those files that have been removed from disk
  48. if (deletedIDs.contains(file.localID)) {
  49. if (file.uploadedFileID != null) {
  50. hasUploadedFiles = true;
  51. await FilesDB.instance.markForDeletion(file.uploadedFileID);
  52. } else {
  53. await FilesDB.instance.deleteLocalFile(file.localID);
  54. }
  55. }
  56. } else {
  57. hasUploadedFiles = true;
  58. await FilesDB.instance.markForDeletion(file.uploadedFileID);
  59. }
  60. await dialog.hide();
  61. }
  62. await FileRepository.instance.reloadFiles();
  63. if (hasUploadedFiles) {
  64. Bus.instance.fire(CollectionUpdatedEvent());
  65. // TODO: Blocking call?
  66. SyncService.instance.deleteFilesOnServer();
  67. }
  68. }
  69. Future<void> deleteFilesOnDeviceOnly(
  70. BuildContext context, List<File> files) async {
  71. final dialog = createProgressDialog(context, "deleting...");
  72. await dialog.show();
  73. final localIDs = List<String>();
  74. for (final file in files) {
  75. if (file.localID != null) {
  76. localIDs.add(file.localID);
  77. }
  78. }
  79. final deletedIDs =
  80. (await PhotoManager.editor.deleteWithIds(localIDs)).toSet();
  81. for (final file in files) {
  82. // Remove only those files that have been removed from disk
  83. if (deletedIDs.contains(file.localID)) {
  84. file.localID = null;
  85. FilesDB.instance.update(file);
  86. }
  87. }
  88. await FileRepository.instance.reloadFiles();
  89. await dialog.hide();
  90. }
  91. void preloadFile(File file) {
  92. if (file.fileType == FileType.video) {
  93. return;
  94. }
  95. if (file.localID == null) {
  96. // getFileFromServer(file);
  97. } else {
  98. if (FileLruCache.get(file) == null) {
  99. file.getAsset().then((asset) {
  100. asset.file.then((assetFile) {
  101. FileLruCache.put(file, assetFile);
  102. });
  103. });
  104. }
  105. }
  106. }
  107. void preloadLocalFileThumbnail(File file) {
  108. if (file.localID == null ||
  109. ThumbnailLruCache.get(file, THUMBNAIL_SMALL_SIZE) != null) {
  110. return;
  111. }
  112. file.getAsset().then((asset) {
  113. asset
  114. .thumbDataWithSize(
  115. THUMBNAIL_SMALL_SIZE,
  116. THUMBNAIL_SMALL_SIZE,
  117. quality: THUMBNAIL_SMALL_SIZE_QUALITY,
  118. )
  119. .then((data) {
  120. ThumbnailLruCache.put(file, THUMBNAIL_SMALL_SIZE, data);
  121. });
  122. });
  123. }
  124. Future<io.File> getNativeFile(File file) async {
  125. if (file.localID == null) {
  126. return getFileFromServer(file);
  127. } else {
  128. return file.getAsset().then((asset) => asset.originFile);
  129. }
  130. }
  131. Future<Uint8List> getBytes(File file, {int quality = 100}) async {
  132. if (file.localID == null) {
  133. return getFileFromServer(file).then((file) => file.readAsBytesSync());
  134. } else {
  135. return await getBytesFromDisk(file, quality: quality);
  136. }
  137. }
  138. Future<Uint8List> getBytesFromDisk(File file, {int quality = 100}) async {
  139. final originalBytes = (await file.getAsset()).originBytes;
  140. if (extension(file.title) == ".HEIC" || quality != 100) {
  141. return originalBytes.then((bytes) {
  142. return FlutterImageCompress.compressWithList(bytes, quality: quality)
  143. .then((converted) {
  144. return Uint8List.fromList(converted);
  145. });
  146. });
  147. } else {
  148. return originalBytes;
  149. }
  150. }
  151. final Map<int, Future<io.File>> fileDownloadsInProgress =
  152. Map<int, Future<io.File>>();
  153. final _thumbnailQueue = LinkedHashMap<int, FileDownloadItem>();
  154. int _currentlyDownloading = 0;
  155. int kMaximumConcurrentDownloads = 32;
  156. class FileDownloadItem {
  157. final File file;
  158. final Completer<io.File> completer;
  159. DownloadStatus status;
  160. FileDownloadItem(
  161. this.file,
  162. this.completer, {
  163. this.status = DownloadStatus.not_started,
  164. });
  165. }
  166. enum DownloadStatus {
  167. not_started,
  168. in_progress,
  169. }
  170. Future<io.File> getFileFromServer(File file,
  171. {ProgressCallback progressCallback}) async {
  172. final cacheManager = file.fileType == FileType.video
  173. ? VideoCacheManager()
  174. : DefaultCacheManager();
  175. return cacheManager.getFileFromCache(file.getDownloadUrl()).then((info) {
  176. if (info == null) {
  177. if (!fileDownloadsInProgress.containsKey(file.uploadedFileID)) {
  178. fileDownloadsInProgress[file.uploadedFileID] = _downloadAndDecrypt(
  179. file,
  180. cacheManager,
  181. progressCallback: progressCallback,
  182. );
  183. }
  184. return fileDownloadsInProgress[file.uploadedFileID];
  185. } else {
  186. return info.file;
  187. }
  188. });
  189. }
  190. Future<io.File> getThumbnailFromServer(File file) async {
  191. return ThumbnailCacheManager()
  192. .getFileFromCache(file.getThumbnailUrl())
  193. .then((info) {
  194. if (info == null) {
  195. if (!_thumbnailQueue.containsKey(file.uploadedFileID)) {
  196. final completer = Completer<io.File>();
  197. _thumbnailQueue[file.uploadedFileID] =
  198. FileDownloadItem(file, completer);
  199. _pollQueue();
  200. return completer.future;
  201. } else {
  202. return _thumbnailQueue[file.uploadedFileID].completer.future;
  203. }
  204. } else {
  205. ThumbnailFileLruCache.put(file, info.file);
  206. return info.file;
  207. }
  208. });
  209. }
  210. void removePendingGetThumbnailRequestIfAny(File file) {
  211. if (_thumbnailQueue[file.uploadedFileID] != null &&
  212. _thumbnailQueue[file.uploadedFileID].status ==
  213. DownloadStatus.not_started) {
  214. _thumbnailQueue.remove(file.uploadedFileID);
  215. }
  216. }
  217. void _pollQueue() async {
  218. if (_thumbnailQueue.length > 0 &&
  219. _currentlyDownloading < kMaximumConcurrentDownloads) {
  220. final firstPendingEntry = _thumbnailQueue.entries.firstWhere(
  221. (entry) => entry.value.status == DownloadStatus.not_started,
  222. orElse: () => null);
  223. if (firstPendingEntry != null) {
  224. final item = firstPendingEntry.value;
  225. _currentlyDownloading++;
  226. item.status = DownloadStatus.in_progress;
  227. try {
  228. final data = await _downloadAndDecryptThumbnail(item.file);
  229. ThumbnailFileLruCache.put(item.file, data);
  230. item.completer.complete(data);
  231. } catch (e) {
  232. _logger.severe("Downloading thumbnail failed", e);
  233. item.completer.completeError(e);
  234. }
  235. _currentlyDownloading--;
  236. _thumbnailQueue.remove(firstPendingEntry.key);
  237. _pollQueue();
  238. }
  239. }
  240. }
  241. Future<io.File> _downloadAndDecrypt(File file, BaseCacheManager cacheManager,
  242. {ProgressCallback progressCallback}) async {
  243. _logger.info("Downloading file " + file.uploadedFileID.toString());
  244. final encryptedFilePath = Configuration.instance.getTempDirectory() +
  245. file.generatedID.toString() +
  246. ".encrypted";
  247. final decryptedFilePath = Configuration.instance.getTempDirectory() +
  248. file.generatedID.toString() +
  249. ".decrypted";
  250. final encryptedFile = io.File(encryptedFilePath);
  251. final decryptedFile = io.File(decryptedFilePath);
  252. final startTime = DateTime.now().millisecondsSinceEpoch;
  253. return Network.instance
  254. .getDio()
  255. .download(
  256. file.getDownloadUrl(),
  257. encryptedFilePath,
  258. options: Options(
  259. headers: {"X-Auth-Token": Configuration.instance.getToken()},
  260. ),
  261. onReceiveProgress: progressCallback,
  262. )
  263. .then((response) async {
  264. if (response.statusCode != 200) {
  265. _logger.warning("Could not download file: ", response.toString());
  266. return null;
  267. } else if (!encryptedFile.existsSync()) {
  268. _logger.warning("File was not downloaded correctly.");
  269. return null;
  270. }
  271. _logger.info("File downloaded: " + file.uploadedFileID.toString());
  272. _logger.info("Download speed: " +
  273. (io.File(encryptedFilePath).lengthSync() /
  274. (DateTime.now().millisecondsSinceEpoch - startTime))
  275. .toString() +
  276. "kBps");
  277. await CryptoUtil.decryptFile(encryptedFilePath, decryptedFilePath,
  278. Sodium.base642bin(file.fileDecryptionHeader), decryptFileKey(file));
  279. _logger.info("File decrypted: " + file.uploadedFileID.toString());
  280. encryptedFile.deleteSync();
  281. var fileExtension = extension(file.title).substring(1).toLowerCase();
  282. var outputFile = decryptedFile;
  283. if (Platform.isAndroid && fileExtension == "heic") {
  284. outputFile = await FlutterImageCompress.compressAndGetFile(
  285. decryptedFilePath,
  286. decryptedFilePath + ".jpg",
  287. keepExif: true,
  288. );
  289. decryptedFile.deleteSync();
  290. }
  291. final cachedFile = await cacheManager.putFile(
  292. file.getDownloadUrl(),
  293. outputFile.readAsBytesSync(),
  294. eTag: file.getDownloadUrl(),
  295. maxAge: Duration(days: 365),
  296. fileExtension: fileExtension,
  297. );
  298. outputFile.deleteSync();
  299. fileDownloadsInProgress.remove(file.uploadedFileID);
  300. return cachedFile;
  301. }).catchError((e) {
  302. fileDownloadsInProgress.remove(file.uploadedFileID);
  303. });
  304. }
  305. Future<io.File> _downloadAndDecryptThumbnail(File file) async {
  306. _logger.info("Downloading thumbnail for " + file.title);
  307. final temporaryPath = Configuration.instance.getTempDirectory() +
  308. file.generatedID.toString() +
  309. "_thumbnail.decrypted";
  310. await Network.instance.getDio().download(
  311. file.getThumbnailUrl(),
  312. temporaryPath,
  313. options: Options(
  314. headers: {"X-Auth-Token": Configuration.instance.getToken()},
  315. ),
  316. );
  317. final encryptedFile = io.File(temporaryPath);
  318. final thumbnailDecryptionKey = decryptFileKey(file);
  319. var data = CryptoUtil.decryptChaCha(
  320. encryptedFile.readAsBytesSync(),
  321. thumbnailDecryptionKey,
  322. Sodium.base642bin(file.thumbnailDecryptionHeader),
  323. );
  324. final thumbnailSize = data.length;
  325. if (thumbnailSize > THUMBNAIL_DATA_LIMIT) {
  326. data = await compressThumbnail(data);
  327. _logger.info("Compressed thumbnail from " +
  328. thumbnailSize.toString() +
  329. " to " +
  330. data.length.toString());
  331. }
  332. encryptedFile.deleteSync();
  333. final cachedThumbnail = ThumbnailCacheManager().putFile(
  334. file.getThumbnailUrl(),
  335. data,
  336. eTag: file.getThumbnailUrl(),
  337. maxAge: Duration(days: 365),
  338. );
  339. return cachedThumbnail;
  340. }
  341. Uint8List decryptFileKey(File file) {
  342. final encryptedKey = Sodium.base642bin(file.encryptedKey);
  343. final nonce = Sodium.base642bin(file.keyDecryptionNonce);
  344. final collectionKey =
  345. CollectionsService.instance.getCollectionKey(file.collectionID);
  346. return CryptoUtil.decryptSync(encryptedKey, collectionKey, nonce);
  347. }
  348. Future<Uint8List> compressThumbnail(Uint8List thumbnail) {
  349. return FlutterImageCompress.compressWithList(
  350. thumbnail,
  351. minHeight: COMPRESSED_THUMBNAIL_RESOLUTION,
  352. minWidth: COMPRESSED_THUMBNAIL_RESOLUTION,
  353. quality: 25,
  354. );
  355. }