Merge pull request #631 from ente-io/imporve_time_parsing

Imporve time parsing from fileName

.github/workflows/code_quality.yml

@@ -29,5 +29,5 @@ jobs:
 
       - name: Run Linter
         run: flutter analyze --no-fatal-infos
-#      - name: Run Test :sed:
-#        run: flutter test
+      - name: Run Test
+        run: flutter test

lib/core/constants.dart

@@ -11,7 +11,7 @@ const String sentryTunnel = "https://sentry-reporter.ente.io";
 const String roadmapURL = "https://roadmap.ente.io";
 const int microSecondsInDay = 86400000000;
 const int android11SDKINT = 30;
-const int jan011991Time = 31580904000000;
+const int jan011981Time = 347135400000000;
 const int galleryLoadStartTime = -8000000000000000; // Wednesday, March 6, 1748
 const int galleryLoadEndTime = 9223372036854775807; // 2^63 -1
 

lib/models/file.dart

@@ -74,24 +74,40 @@ class File extends EnteFile {
     file.deviceFolder = pathName;
     file.location = Location(asset.latitude, asset.longitude);
     file.fileType = _fileTypeFromAsset(asset);
-    file.creationTime = asset.createDateTime.microsecondsSinceEpoch;
-    if (file.creationTime == null || (file.creationTime! <= jan011991Time)) {
-      try {
-        final parsedDateTime =
-            parseDateFromFileName(basenameWithoutExtension(file.title ?? ""));
-
-        file.creationTime = parsedDateTime?.microsecondsSinceEpoch ??
-            asset.modifiedDateTime.microsecondsSinceEpoch;
-      } catch (e) {
-        file.creationTime = asset.modifiedDateTime.microsecondsSinceEpoch;
-      }
-    }
+    file.creationTime = parseFileCreationTime(file.title, asset);
     file.modificationTime = asset.modifiedDateTime.microsecondsSinceEpoch;
     file.fileSubType = asset.subtype;
     file.metadataVersion = kCurrentMetadataVersion;
     return file;
   }
 
+  static int parseFileCreationTime(String? fileTitle, AssetEntity asset) {
+    int creationTime = asset.createDateTime.microsecondsSinceEpoch;
+    if (creationTime >= jan011981Time) {
+      // assuming that fileSystem is returning correct creationTime.
+      // During upload, this might get overridden with exif Creation time
+      return creationTime;
+    } else {
+      if (asset.modifiedDateTime.microsecondsSinceEpoch >= jan011981Time) {
+        creationTime = asset.modifiedDateTime.microsecondsSinceEpoch;
+      } else {
+        creationTime = DateTime.now().toUtc().microsecondsSinceEpoch;
+      }
+      try {
+        final parsedDateTime = parseDateTimeFromFileNameV2(
+          basenameWithoutExtension(fileTitle ?? ""),
+        );
+        if (parsedDateTime != null) {
+          creationTime = parsedDateTime.microsecondsSinceEpoch;
+        }
+      } catch (e) {
+        // ignore
+      }
+    }
+
+    return creationTime;
+  }
+
   static FileType _fileTypeFromAsset(AssetEntity asset) {
     FileType type = FileType.image;
     switch (asset.type) {

lib/utils/date_time_util.dart

@@ -1,3 +1,4 @@
+import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:intl/intl.dart';
 
@@ -269,6 +270,7 @@ bool isValidDate({
   return true;
 }
 
+@Deprecated("Use parseDateTimeV2 ")
 DateTime? parseDateFromFileName(String fileName) {
   if (fileName.startsWith('IMG-') || fileName.startsWith('VID-')) {
 // Whatsapp media files
@@ -288,3 +290,44 @@ DateTime? parseDateFromFileName(String fileName) {
     );
   }
 }
+
+final RegExp exp = RegExp('[A-Za-z]');
+
+DateTime? parseDateTimeFromFileNameV2(String fileName) {
+  String val = fileName.replaceAll(exp, '');
+  if (val.isNotEmpty && !isNumeric(val[0])) {
+    val = val.substring(1, val.length);
+  }
+  if (val.isNotEmpty && !isNumeric(val[val.length - 1])) {
+    val = val.substring(0, val.length - 1);
+  }
+  final int countOfHyphen = val.split("-").length - 1;
+  final int countUnderScore = val.split("_").length - 1;
+  String valForParser = val;
+  if (countOfHyphen == 1) {
+    valForParser = val.replaceAll("-", "T");
+  } else if (countUnderScore == 1 || countUnderScore == 2) {
+    valForParser = val.replaceFirst("_", "T");
+    if (countUnderScore == 2) {
+      valForParser = valForParser.split("_")[0];
+    }
+  } else if (countOfHyphen == 2) {
+    valForParser = val.replaceAll(".", ":");
+  } else if (countOfHyphen == 6) {
+    final splits = val.split("-");
+    valForParser =
+        "${splits[0]}${splits[1]}${splits[2]}T${splits[3]}${splits[4]}${splits[5]}";
+  }
+  final result = DateTime.tryParse(valForParser);
+  if (kDebugMode && result == null) {
+    debugPrint("Failed to parse $fileName dateTime from $valForParser");
+  }
+  return result;
+}
+
+bool isNumeric(String? s) {
+  if (s == null) {
+    return false;
+  }
+  return double.tryParse(s) != null;
+}

lib/utils/share_util.dart

@@ -99,7 +99,7 @@ Future<List<File>> convertIncomingSharedMediaToFile(
     }
     if (enteFile.creationTime == null || enteFile.creationTime == 0) {
       final parsedDateTime =
-          parseDateFromFileName(basenameWithoutExtension(media.path));
+          parseDateTimeFromFileNameV2(basenameWithoutExtension(media.path));
       if (parsedDateTime != null) {
         enteFile.creationTime = parsedDateTime.microsecondsSinceEpoch;
       } else {

pubspec.lock

@@ -1,6 +1,13 @@
 # Generated by pub
 # See https://dart.dev/tools/pub/glossary#lockfile
 packages:
+  _fe_analyzer_shared:
+    dependency: transitive
+    description:
+      name: _fe_analyzer_shared
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "47.0.0"
   adaptive_theme:
     dependency: "direct main"
     description:
@@ -8,6 +15,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "3.1.1"
+  analyzer:
+    dependency: transitive
+    description:
+      name: analyzer
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "4.7.0"
   animate_do:
     dependency: "direct main"
     description:
@@ -162,6 +176,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "3.0.2"
+  coverage:
+    dependency: transitive
+    description:
+      name: coverage
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.3.2"
   crypto:
     dependency: transitive
     description:
@@ -546,6 +567,20 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "8.0.9"
+  frontend_server_client:
+    dependency: transitive
+    description:
+      name: frontend_server_client
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.1.3"
+  glob:
+    dependency: transitive
+    description:
+      name: glob
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.1.1"
   google_nav_bar:
     dependency: "direct main"
     description:
@@ -581,6 +616,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.0.3"
+  http_multi_server:
+    dependency: transitive
+    description:
+      name: http_multi_server
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "3.2.1"
   http_parser:
     dependency: transitive
     description:
@@ -644,6 +686,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.17.0"
+  io:
+    dependency: transitive
+    description:
+      name: io
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.0.3"
   js:
     dependency: transitive
     description:
@@ -774,6 +823,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.0.0"
+  node_preamble:
+    dependency: transitive
+    description:
+      name: node_preamble
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.0.1"
   octo_image:
     dependency: transitive
     description:
@@ -788,6 +844,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.4.5"
+  package_config:
+    dependency: transitive
+    description:
+      name: package_config
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.1.0"
   package_info_plus:
     dependency: "direct main"
     description:
@@ -970,6 +1033,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "3.6.2"
+  pool:
+    dependency: transitive
+    description:
+      name: pool
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.5.1"
   process:
     dependency: transitive
     description:
@@ -984,6 +1054,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "6.0.3"
+  pub_semver:
+    dependency: transitive
+    description:
+      name: pub_semver
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.1.2"
   quiver:
     dependency: "direct main"
     description:
@@ -1124,11 +1201,53 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "2.1.1"
+  shelf:
+    dependency: transitive
+    description:
+      name: shelf
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.4.0"
+  shelf_packages_handler:
+    dependency: transitive
+    description:
+      name: shelf_packages_handler
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "3.0.1"
+  shelf_static:
+    dependency: transitive
+    description:
+      name: shelf_static
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.1.1"
+  shelf_web_socket:
+    dependency: transitive
+    description:
+      name: shelf_web_socket
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.0.3"
   sky_engine:
     dependency: transitive
     description: flutter
     source: sdk
     version: "0.0.99"
+  source_map_stack_trace:
+    dependency: transitive
+    description:
+      name: source_map_stack_trace
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.1.1"
+  source_maps:
+    dependency: transitive
+    description:
+      name: source_maps
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.10.10"
   source_span:
     dependency: transitive
     description:
@@ -1220,6 +1339,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.2.0"
+  test:
+    dependency: "direct dev"
+    description:
+      name: test
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.21.1"
   test_api:
     dependency: transitive
     description:
@@ -1227,6 +1353,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.4.9"
+  test_core:
+    dependency: transitive
+    description:
+      name: test_core
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.4.13"
   timezone:
     dependency: transitive
     description:
@@ -1381,6 +1514,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.3.3"
+  vm_service:
+    dependency: transitive
+    description:
+      name: vm_service
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "8.3.0"
   wakelock:
     dependency: "direct main"
     description:
@@ -1423,6 +1563,27 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.0.2"
+  watcher:
+    dependency: transitive
+    description:
+      name: watcher
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.0.2"
+  web_socket_channel:
+    dependency: transitive
+    description:
+      name: web_socket_channel
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "2.2.0"
+  webkit_inspection_protocol:
+    dependency: transitive
+    description:
+      name: webkit_inspection_protocol
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "1.2.0"
   win32:
     dependency: transitive
     description:

pubspec.yaml

@@ -127,6 +127,7 @@ dev_dependencies:
   flutter_lints: ^2.0.1
   flutter_test:
     sdk: flutter
+  test:
 
 flutter_icons:
   android: "launcher_icon"

test/utils/date_time_util_test.dart

@@ -0,0 +1,40 @@
+import 'package:flutter/foundation.dart';
+import 'package:photos/core/constants.dart';
+import 'package:photos/utils/date_time_util.dart';
+import 'package:test/test.dart';
+
+void main() {
+  test("parseDateTimeFromFile", () {
+    final List<String> validParsing = [
+      "IMG-20221109-WA0000",
+      '''Screenshot_20220807-195908_Firefox''',
+      '''Screenshot_20220507-195908''',
+      "2019-02-18 16.00.12-DCMX",
+      "20221107_231730",
+      "2020-11-01 02.31.02",
+      "IMG_20210921_144423",
+      "2019-10-31 155703",
+      "IMG_20210921_144423_783",
+      "Screenshot_2022-06-21-16-51-29-164_newFormat",
+    ];
+    for (String val in validParsing) {
+      final parsedValue = parseDateTimeFromFileNameV2(val);
+      expect(
+        parsedValue != null,
+        true,
+        reason: "Failed to parse time from $val",
+      );
+      if (kDebugMode) {
+        debugPrint("Parsed $val as ${parsedValue?.toIso8601String()}");
+      }
+    }
+  });
+
+  test("verify constants", () {
+    expect(
+      jan011981Time,
+      DateTime(1981, 1, 1).toUtc().microsecondsSinceEpoch,
+      reason: "constant mismatch",
+    );
+  });
+}