Remove Isar DB for Embeddings

This commit is contained in:
vishnukvmd 2024-04-30 19:52:30 +05:30
parent c0e12e84ef
commit 348b784674
5 changed files with 17 additions and 1148 deletions

View file

@ -1,79 +0,0 @@
import "dart:io";
import "package:isar/isar.dart";
import 'package:path_provider/path_provider.dart';
import "package:photos/core/event_bus.dart";
import "package:photos/events/embedding_updated_event.dart";
import "package:photos/models/embedding.dart";
class EmbeddingsDB {
late final Isar _isar;
EmbeddingsDB._privateConstructor();
static final EmbeddingsDB instance = EmbeddingsDB._privateConstructor();
Future<void> init() async {
final dir = await getApplicationDocumentsDirectory();
_isar = await Isar.open(
[EmbeddingSchema],
directory: dir.path,
);
await _clearDeprecatedStore(dir);
}
Future<void> clearTable() async {
await _isar.writeTxn(() => _isar.clear());
}
Future<List<Embedding>> getAll(Model model) async {
return _isar.embeddings.filter().modelEqualTo(model).findAll();
}
Future<void> put(Embedding embedding) {
return _isar.writeTxn(() async {
await _isar.embeddings.putByIndex(Embedding.index, embedding);
Bus.instance.fire(EmbeddingUpdatedEvent());
});
}
Future<void> putMany(List<Embedding> embeddings) {
return _isar.writeTxn(() async {
await _isar.embeddings.putAllByIndex(Embedding.index, embeddings);
Bus.instance.fire(EmbeddingUpdatedEvent());
});
}
Future<List<Embedding>> getUnsyncedEmbeddings() async {
return await _isar.embeddings.filter().updationTimeEqualTo(null).findAll();
}
Future<void> deleteEmbeddings(List<int> fileIDs) async {
await _isar.writeTxn(() async {
final embeddings = <Embedding>[];
for (final fileID in fileIDs) {
embeddings.addAll(
await _isar.embeddings.filter().fileIDEqualTo(fileID).findAll(),
);
}
await _isar.embeddings.deleteAll(embeddings.map((e) => e.id).toList());
Bus.instance.fire(EmbeddingUpdatedEvent());
});
}
Future<void> deleteAllForModel(Model model) async {
await _isar.writeTxn(() async {
final embeddings =
await _isar.embeddings.filter().modelEqualTo(model).findAll();
await _isar.embeddings.deleteAll(embeddings.map((e) => e.id).toList());
Bus.instance.fire(EmbeddingUpdatedEvent());
});
}
Future<void> _clearDeprecatedStore(Directory dir) async {
final deprecatedStore = Directory(dir.path + "/object-box-store");
if (await deprecatedStore.exists()) {
await deprecatedStore.delete(recursive: true);
}
}
}

View file

@ -27,6 +27,11 @@ class EmbeddingsDB {
return _dbFuture!; return _dbFuture!;
} }
Future<void> init() async {
final dir = await getApplicationDocumentsDirectory();
await _clearDeprecatedStores(dir);
}
Future<SqliteDatabase> _initDatabase() async { Future<SqliteDatabase> _initDatabase() async {
final Directory documentsDirectory = final Directory documentsDirectory =
await getApplicationDocumentsDirectory(); await getApplicationDocumentsDirectory();
@ -126,4 +131,15 @@ class EmbeddingsDB {
embedding.updationTime, embedding.updationTime,
]; ];
} }
Future<void> _clearDeprecatedStores(Directory dir) async {
final deprecatedStore = Directory(dir.path + "/object-box-store");
if (await deprecatedStore.exists()) {
await deprecatedStore.delete(recursive: true);
}
final deprecatedDB = File(dir.path + "/default.isar");
if (await deprecatedDB.exists()) {
await deprecatedDB.delete();
}
}
} }

View file

@ -1,17 +1,7 @@
import "dart:convert"; import "dart:convert";
import "package:isar/isar.dart";
part 'embedding.g.dart';
@collection
class Embedding { class Embedding {
static const index = 'unique_file_model_embedding';
Id id = Isar.autoIncrement;
final int fileID; final int fileID;
@enumerated
@Index(name: index, composite: [CompositeIndex('fileID')], unique: true, replace: true)
final Model model; final Model model;
final List<double> embedding; final List<double> embedding;
int? updationTime; int? updationTime;

File diff suppressed because it is too large Load diff

View file

@ -73,6 +73,7 @@ class SemanticSearchService {
? ONNX(shouldDownloadOverMobileData) ? ONNX(shouldDownloadOverMobileData)
: GGML(shouldDownloadOverMobileData); : GGML(shouldDownloadOverMobileData);
await EmbeddingStore.instance.init(); await EmbeddingStore.instance.init();
await EmbeddingsDB.instance.init();
await _loadEmbeddings(); await _loadEmbeddings();
Bus.instance.on<EmbeddingUpdatedEvent>().listen((event) { Bus.instance.on<EmbeddingUpdatedEvent>().listen((event) {
_embeddingLoaderDebouncer.run(() async { _embeddingLoaderDebouncer.run(() async {