diff --git a/lib/db/memories_db.dart b/lib/db/memories_db.dart
index 098750f79..de894fe6a 100644
--- a/lib/db/memories_db.dart
+++ b/lib/db/memories_db.dart
@@ -59,9 +59,9 @@ class MemoriesDB {
         conflictAlgorithm: ConflictAlgorithm.replace);
   }
 
-  Future<Set<int>> getSeenFileIDs() async {
+  Future<Map<int, int>> getSeenTimes() async {
     final db = await instance.database;
-    return _convertToFileIDs(await db.query(table));
+    return _convertToSeenTimes(await db.query(table));
   }
 
   Map<String, dynamic> _getRowForSeenMemory(Memory memory, int timestamp) {
@@ -71,11 +71,11 @@ class MemoriesDB {
     return row;
   }
 
-  Set<int> _convertToFileIDs(List<Map<String, dynamic>> rows) {
-    final fileIDs = Set<int>();
+  Map<int, int> _convertToSeenTimes(List<Map<String, dynamic>> rows) {
+    final seenTimes = Map<int, int>();
     for (final row in rows) {
-      fileIDs.add(row[columnFileID]);
+      seenTimes[row[columnFileID]] = int.parse(row[columnSeenTime]);
     }
-    return fileIDs;
+    return seenTimes;
   }
 }
diff --git a/lib/memories_service.dart b/lib/memories_service.dart
index 3b0089f0d..68e97309d 100644
--- a/lib/memories_service.dart
+++ b/lib/memories_service.dart
@@ -53,11 +53,12 @@ class MemoriesService extends ChangeNotifier {
                 DateTime.fromMicrosecondsSinceEpoch(endCreationTime)));
       files.addAll(filesInYear);
     }
-    final seenFileIDs = await _memoriesDB.getSeenFileIDs();
+    final seenTimes = await _memoriesDB.getSeenTimes();
     final memories = List<Memory>();
     for (final file in files) {
       if (filter.shouldInclude(file)) {
-        memories.add(Memory(file, seenFileIDs.contains(file.generatedId)));
+        final seenTime = seenTimes[file.generatedId] ?? -1;
+        memories.add(Memory(file, seenTime));
       }
     }
     _logger.info("Number of memories: " + memories.length.toString());
@@ -76,6 +77,7 @@ class MemoriesService extends ChangeNotifier {
   }
 
   Future markMemoryAsSeen(Memory memory) async {
+    _logger.info("Marking memory " + memory.file.title + " as seen");
     await _memoriesDB.markMemoryAsSeen(
         memory, DateTime.now().microsecondsSinceEpoch);
     notifyListeners();
diff --git a/lib/models/memory.dart b/lib/models/memory.dart
index ff1a0e621..21e503ab9 100644
--- a/lib/models/memory.dart
+++ b/lib/models/memory.dart
@@ -2,15 +2,19 @@ import 'package:photos/models/file.dart';
 
 class Memory {
   final File file;
-  bool _isSeen;
+  int _seenTime;
 
-  Memory(this.file, this._isSeen);
+  Memory(this.file, this._seenTime);
 
   bool isSeen() {
-    return _isSeen;
+    return _seenTime != -1;
   }
 
-  bool markSeen() {
-    _isSeen = true;
+  int seenTime() {
+    return _seenTime;
+  }
+
+  void markSeen() {
+    _seenTime = DateTime.now().microsecondsSinceEpoch;
   }
 }
diff --git a/lib/ui/memories_widget.dart b/lib/ui/memories_widget.dart
index a34d47a3e..0be77c721 100644
--- a/lib/ui/memories_widget.dart
+++ b/lib/ui/memories_widget.dart
@@ -152,6 +152,10 @@ class MemoryWidget extends StatelessWidget {
       if (!memories[index].isSeen()) {
         return index;
       }
+      if (index > 0 &&
+          memories[index - 1].seenTime() > memories[index].seenTime()) {
+        return index;
+      }
     }
     return 0;
   }
@@ -273,6 +277,7 @@ class _FullScreenMemoryState extends State<FullScreenMemory> {
       control: SwiperControl(
         color: _opacity == 1 ? Colors.white54 : Colors.transparent,
       ),
+      index: _index,
       onIndexChanged: (index) async {
         await MemoriesService.instance.markMemoryAsSeen(widget.memories[index]);
         setState(() {