Remove Isar DB for Embeddings
This commit is contained in:
parent
c0e12e84ef
commit
348b784674
5 changed files with 17 additions and 1148 deletions
|
@ -1,79 +0,0 @@
|
||||||
import "dart:io";
|
|
||||||
|
|
||||||
import "package:isar/isar.dart";
|
|
||||||
import 'package:path_provider/path_provider.dart';
|
|
||||||
import "package:photos/core/event_bus.dart";
|
|
||||||
import "package:photos/events/embedding_updated_event.dart";
|
|
||||||
import "package:photos/models/embedding.dart";
|
|
||||||
|
|
||||||
class EmbeddingsDB {
|
|
||||||
late final Isar _isar;
|
|
||||||
|
|
||||||
EmbeddingsDB._privateConstructor();
|
|
||||||
|
|
||||||
static final EmbeddingsDB instance = EmbeddingsDB._privateConstructor();
|
|
||||||
|
|
||||||
Future<void> init() async {
|
|
||||||
final dir = await getApplicationDocumentsDirectory();
|
|
||||||
_isar = await Isar.open(
|
|
||||||
[EmbeddingSchema],
|
|
||||||
directory: dir.path,
|
|
||||||
);
|
|
||||||
await _clearDeprecatedStore(dir);
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> clearTable() async {
|
|
||||||
await _isar.writeTxn(() => _isar.clear());
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<List<Embedding>> getAll(Model model) async {
|
|
||||||
return _isar.embeddings.filter().modelEqualTo(model).findAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> put(Embedding embedding) {
|
|
||||||
return _isar.writeTxn(() async {
|
|
||||||
await _isar.embeddings.putByIndex(Embedding.index, embedding);
|
|
||||||
Bus.instance.fire(EmbeddingUpdatedEvent());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> putMany(List<Embedding> embeddings) {
|
|
||||||
return _isar.writeTxn(() async {
|
|
||||||
await _isar.embeddings.putAllByIndex(Embedding.index, embeddings);
|
|
||||||
Bus.instance.fire(EmbeddingUpdatedEvent());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<List<Embedding>> getUnsyncedEmbeddings() async {
|
|
||||||
return await _isar.embeddings.filter().updationTimeEqualTo(null).findAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> deleteEmbeddings(List<int> fileIDs) async {
|
|
||||||
await _isar.writeTxn(() async {
|
|
||||||
final embeddings = <Embedding>[];
|
|
||||||
for (final fileID in fileIDs) {
|
|
||||||
embeddings.addAll(
|
|
||||||
await _isar.embeddings.filter().fileIDEqualTo(fileID).findAll(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
await _isar.embeddings.deleteAll(embeddings.map((e) => e.id).toList());
|
|
||||||
Bus.instance.fire(EmbeddingUpdatedEvent());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> deleteAllForModel(Model model) async {
|
|
||||||
await _isar.writeTxn(() async {
|
|
||||||
final embeddings =
|
|
||||||
await _isar.embeddings.filter().modelEqualTo(model).findAll();
|
|
||||||
await _isar.embeddings.deleteAll(embeddings.map((e) => e.id).toList());
|
|
||||||
Bus.instance.fire(EmbeddingUpdatedEvent());
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Future<void> _clearDeprecatedStore(Directory dir) async {
|
|
||||||
final deprecatedStore = Directory(dir.path + "/object-box-store");
|
|
||||||
if (await deprecatedStore.exists()) {
|
|
||||||
await deprecatedStore.delete(recursive: true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -27,6 +27,11 @@ class EmbeddingsDB {
|
||||||
return _dbFuture!;
|
return _dbFuture!;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Future<void> init() async {
|
||||||
|
final dir = await getApplicationDocumentsDirectory();
|
||||||
|
await _clearDeprecatedStores(dir);
|
||||||
|
}
|
||||||
|
|
||||||
Future<SqliteDatabase> _initDatabase() async {
|
Future<SqliteDatabase> _initDatabase() async {
|
||||||
final Directory documentsDirectory =
|
final Directory documentsDirectory =
|
||||||
await getApplicationDocumentsDirectory();
|
await getApplicationDocumentsDirectory();
|
||||||
|
@ -126,4 +131,15 @@ class EmbeddingsDB {
|
||||||
embedding.updationTime,
|
embedding.updationTime,
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Future<void> _clearDeprecatedStores(Directory dir) async {
|
||||||
|
final deprecatedStore = Directory(dir.path + "/object-box-store");
|
||||||
|
if (await deprecatedStore.exists()) {
|
||||||
|
await deprecatedStore.delete(recursive: true);
|
||||||
|
}
|
||||||
|
final deprecatedDB = File(dir.path + "/default.isar");
|
||||||
|
if (await deprecatedDB.exists()) {
|
||||||
|
await deprecatedDB.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,17 +1,7 @@
|
||||||
import "dart:convert";
|
import "dart:convert";
|
||||||
|
|
||||||
import "package:isar/isar.dart";
|
|
||||||
|
|
||||||
part 'embedding.g.dart';
|
|
||||||
|
|
||||||
@collection
|
|
||||||
class Embedding {
|
class Embedding {
|
||||||
static const index = 'unique_file_model_embedding';
|
|
||||||
|
|
||||||
Id id = Isar.autoIncrement;
|
|
||||||
final int fileID;
|
final int fileID;
|
||||||
@enumerated
|
|
||||||
@Index(name: index, composite: [CompositeIndex('fileID')], unique: true, replace: true)
|
|
||||||
final Model model;
|
final Model model;
|
||||||
final List<double> embedding;
|
final List<double> embedding;
|
||||||
int? updationTime;
|
int? updationTime;
|
||||||
|
|
File diff suppressed because it is too large
Load diff
|
@ -73,6 +73,7 @@ class SemanticSearchService {
|
||||||
? ONNX(shouldDownloadOverMobileData)
|
? ONNX(shouldDownloadOverMobileData)
|
||||||
: GGML(shouldDownloadOverMobileData);
|
: GGML(shouldDownloadOverMobileData);
|
||||||
await EmbeddingStore.instance.init();
|
await EmbeddingStore.instance.init();
|
||||||
|
await EmbeddingsDB.instance.init();
|
||||||
await _loadEmbeddings();
|
await _loadEmbeddings();
|
||||||
Bus.instance.on<EmbeddingUpdatedEvent>().listen((event) {
|
Bus.instance.on<EmbeddingUpdatedEvent>().listen((event) {
|
||||||
_embeddingLoaderDebouncer.run(() async {
|
_embeddingLoaderDebouncer.run(() async {
|
||||||
|
|
Loading…
Reference in a new issue