0ct0pu5/ente
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Clear embeddings for files that no longer exist locally (#1734)

Browse source
This commit is contained in:
Vishnu Mohandas 2024-02-20 16:22:08 +05:30 • committed by GitHub
commit 712fb85eec
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 32 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
lib/db/embeddings_db.dart
View file

@ -48,6 +48,19 @@ class EmbeddingsDB {
return await _isar.embeddings.filter().updationTimeEqualTo(null).findAll();
}
Future<void> deleteEmbeddings(List<int> fileIDs) async {
await _isar.writeTxn(() async {
final embeddings = <Embedding>[];
for (final fileID in fileIDs) {
embeddings.addAll(
await _isar.embeddings.filter().fileIDEqualTo(fileID).findAll(),
);
}
await _isar.embeddings.deleteAll(embeddings.map((e) => e.id).toList());
Bus.instance.fire(EmbeddingUpdatedEvent());
});
}
Future<void> deleteAllForModel(Model model) async {
await _isar.writeTxn(() async {
final embeddings =

11
lib/services/machine_learning/semantic_search/embedding_store.dart
View file

@ -53,13 +53,22 @@ class EmbeddingStore {
final fileMap = await FilesDB.instance
.getFilesFromIDs(pendingItems.map((e) => e.fileID).toList());
_logger.info("Pushing ${pendingItems.length} embeddings");
final deletedEntries = <int>[];
for (final item in pendingItems) {
try {
await _pushEmbedding(fileMap[item.fileID]!, item);
final file = fileMap[item.fileID];
if (file != null) {
await _pushEmbedding(file, item);
} else {
deletedEntries.add(item.fileID);
}
} catch (e, s) {
_logger.severe(e, s);
}
}
if (deletedEntries.isNotEmpty) {
await EmbeddingsDB.instance.deleteEmbeddings(deletedEntries);
}
}
Future<void> storeEmbedding(EnteFile file, Embedding embedding) async {

10
lib/services/machine_learning/semantic_search/semantic_search_service.dart
View file

@ -243,15 +243,23 @@ class SemanticSearchService {
final ignoredCollections =
CollectionsService.instance.getHiddenCollectionIds();
final deletedEntries = <int>[];
for (final result in queryResults) {
final file = filesMap[result.id];
if (file != null && !ignoredCollections.contains(file.collectionID)) {
results.add(filesMap[result.id]!);
results.add(file);
}
if (file == null) {
deletedEntries.add(result.id);
}
}
_logger.info(results.length.toString() + " results");
if (deletedEntries.isNotEmpty) {
unawaited(EmbeddingsDB.instance.deleteEmbeddings(deletedEntries));
}
return results;
}