Compare commits
5 commits
main
...
dev/fix-sy
Author | SHA1 | Date | |
---|---|---|---|
|
de7475f3fc | ||
|
7f5d554d1a | ||
|
e033e9aad2 | ||
|
061c567468 | ||
|
4ce3676127 |
13 changed files with 128 additions and 52 deletions
6
cli/src/api/open-api/api.ts
generated
6
cli/src/api/open-api/api.ts
generated
|
@ -959,6 +959,12 @@ export interface AuditDeletesResponseDto {
|
|||
* @memberof AuditDeletesResponseDto
|
||||
*/
|
||||
'needsFullSync': boolean;
|
||||
/**
|
||||
*
|
||||
* @type {string}
|
||||
* @memberof AuditDeletesResponseDto
|
||||
*/
|
||||
'requestedAt'?: string;
|
||||
}
|
||||
/**
|
||||
*
|
||||
|
|
|
@ -48,23 +48,28 @@ class AssetService {
|
|||
return changes;
|
||||
}
|
||||
|
||||
/// Returns `(null, null)` if changes are invalid -> requires full sync
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete)>
|
||||
/// Returns `(null, null, time)` if changes are invalid -> requires full sync
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
|
||||
_getRemoteAssetChanges(User user, DateTime since) async {
|
||||
final deleted = await _apiService.auditApi
|
||||
.getAuditDeletes(EntityType.ASSET, since, userId: user.id);
|
||||
if (deleted == null || deleted.needsFullSync) return (null, null);
|
||||
if (deleted == null || deleted.needsFullSync) {
|
||||
return (null, null, deleted?.requestedAt);
|
||||
}
|
||||
final assetDto = await _apiService.assetApi
|
||||
.getAllAssets(userId: user.id, updatedAfter: since);
|
||||
if (assetDto == null) return (null, null);
|
||||
return (assetDto.map(Asset.remote).toList(), deleted.ids);
|
||||
if (assetDto == null) return (null, null, deleted.requestedAt);
|
||||
return (
|
||||
assetDto.map(Asset.remote).toList(),
|
||||
deleted.ids,
|
||||
deleted.requestedAt
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns `null` if the server state did not change, else list of assets
|
||||
Future<List<Asset>?> _getRemoteAssets(User user) async {
|
||||
Future<List<Asset>?> _getRemoteAssets(User user, DateTime now) async {
|
||||
const int chunkSize = 5000;
|
||||
try {
|
||||
final DateTime now = DateTime.now().toUtc();
|
||||
final List<Asset> allAssets = [];
|
||||
for (int i = 0;; i += chunkSize) {
|
||||
final List<AssetResponseDto>? assets =
|
||||
|
|
|
@ -41,17 +41,20 @@ class SyncService {
|
|||
/// Returns `true` if there were any changes
|
||||
Future<bool> syncRemoteAssetsToDb(
|
||||
User user,
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
|
||||
Function(
|
||||
User user,
|
||||
DateTime since,
|
||||
) getChangedAssets,
|
||||
FutureOr<List<Asset>?> Function(User user) loadAssets,
|
||||
FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
|
||||
) =>
|
||||
_lock.run(
|
||||
() async =>
|
||||
await _syncRemoteAssetChanges(user, getChangedAssets) ??
|
||||
await _syncRemoteAssetsFull(user, loadAssets),
|
||||
);
|
||||
_lock.run(() async {
|
||||
final (changes, serverTime) =
|
||||
await _syncRemoteAssetChanges(user, getChangedAssets);
|
||||
if (changes != null) return changes;
|
||||
final time = serverTime ?? DateTime.now().toUtc();
|
||||
return await _syncRemoteAssetsFull(user, time, loadAssets);
|
||||
});
|
||||
|
||||
/// Syncs remote albums to the database
|
||||
/// returns `true` if there were any changes
|
||||
|
@ -146,19 +149,22 @@ class SyncService {
|
|||
return true;
|
||||
}
|
||||
|
||||
/// Efficiently syncs assets via changes. Returns `null` when a full sync is required.
|
||||
Future<bool?> _syncRemoteAssetChanges(
|
||||
/// Efficiently syncs assets via changes. Returns `(null, serverTime)` when a full sync is required.
|
||||
Future<(bool?, DateTime? serverTime)> _syncRemoteAssetChanges(
|
||||
User user,
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete)> Function(
|
||||
Future<(List<Asset>? toUpsert, List<String>? toDelete, DateTime? time)>
|
||||
Function(
|
||||
User user,
|
||||
DateTime since,
|
||||
) getChangedAssets,
|
||||
) async {
|
||||
final DateTime? since = _db.eTags.getByIdSync(user.id)?.time?.toUtc();
|
||||
if (since == null) return null;
|
||||
final DateTime now = DateTime.now();
|
||||
final (toUpsert, toDelete) = await getChangedAssets(user, since);
|
||||
if (toUpsert == null || toDelete == null) return null;
|
||||
final DateTime now = DateTime.now().toUtc();
|
||||
final (toUpsert, toDelete, serverTime) =
|
||||
await getChangedAssets(user, since ?? now);
|
||||
if (since == null || toUpsert == null || toDelete == null) {
|
||||
return (null, serverTime);
|
||||
}
|
||||
try {
|
||||
if (toDelete.isNotEmpty) {
|
||||
await handleRemoteAssetRemoval(toDelete);
|
||||
|
@ -168,14 +174,14 @@ class SyncService {
|
|||
await upsertAssetsWithExif(updated);
|
||||
}
|
||||
if (toUpsert.isNotEmpty || toDelete.isNotEmpty) {
|
||||
await _updateUserAssetsETag(user, now);
|
||||
return true;
|
||||
await _updateUserAssetsETag(user, serverTime ?? now);
|
||||
return (true, serverTime);
|
||||
}
|
||||
return false;
|
||||
return (false, serverTime);
|
||||
} on IsarError catch (e) {
|
||||
_log.severe("Failed to sync remote assets to db: $e");
|
||||
}
|
||||
return null;
|
||||
return (null, serverTime);
|
||||
}
|
||||
|
||||
/// Deletes remote-only assets, updates merged assets to be local-only
|
||||
|
@ -202,11 +208,11 @@ class SyncService {
|
|||
|
||||
/// Syncs assets by loading and comparing all assets from the server.
|
||||
Future<bool> _syncRemoteAssetsFull(
|
||||
User user,
|
||||
FutureOr<List<Asset>?> Function(User user) loadAssets,
|
||||
final User user,
|
||||
final DateTime now,
|
||||
final FutureOr<List<Asset>?> Function(User user, DateTime now) loadAssets,
|
||||
) async {
|
||||
final DateTime now = DateTime.now().toUtc();
|
||||
final List<Asset>? remote = await loadAssets(user);
|
||||
final List<Asset>? remote = await loadAssets(user, now);
|
||||
if (remote == null) {
|
||||
return false;
|
||||
}
|
||||
|
|
1
mobile/openapi/doc/AuditDeletesResponseDto.md
generated
1
mobile/openapi/doc/AuditDeletesResponseDto.md
generated
|
@ -10,6 +10,7 @@ Name | Type | Description | Notes
|
|||
------------ | ------------- | ------------- | -------------
|
||||
**ids** | **List<String>** | | [default to const []]
|
||||
**needsFullSync** | **bool** | |
|
||||
**requestedAt** | [**DateTime**](DateTime.md) | | [optional]
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
||||
|
|
|
@ -15,30 +15,46 @@ class AuditDeletesResponseDto {
|
|||
AuditDeletesResponseDto({
|
||||
this.ids = const [],
|
||||
required this.needsFullSync,
|
||||
this.requestedAt,
|
||||
});
|
||||
|
||||
List<String> ids;
|
||||
|
||||
bool needsFullSync;
|
||||
|
||||
///
|
||||
/// Please note: This property should have been non-nullable! Since the specification file
|
||||
/// does not include a default value (using the "default:" property), however, the generated
|
||||
/// source code must fall back to having a nullable type.
|
||||
/// Consider adding a "default:" property in the specification file to hide this note.
|
||||
///
|
||||
DateTime? requestedAt;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) => identical(this, other) || other is AuditDeletesResponseDto &&
|
||||
other.ids == ids &&
|
||||
other.needsFullSync == needsFullSync;
|
||||
other.needsFullSync == needsFullSync &&
|
||||
other.requestedAt == requestedAt;
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
// ignore: unnecessary_parenthesis
|
||||
(ids.hashCode) +
|
||||
(needsFullSync.hashCode);
|
||||
(needsFullSync.hashCode) +
|
||||
(requestedAt == null ? 0 : requestedAt!.hashCode);
|
||||
|
||||
@override
|
||||
String toString() => 'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync]';
|
||||
String toString() => 'AuditDeletesResponseDto[ids=$ids, needsFullSync=$needsFullSync, requestedAt=$requestedAt]';
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final json = <String, dynamic>{};
|
||||
json[r'ids'] = this.ids;
|
||||
json[r'needsFullSync'] = this.needsFullSync;
|
||||
if (this.requestedAt != null) {
|
||||
json[r'requestedAt'] = this.requestedAt!.toUtc().toIso8601String();
|
||||
} else {
|
||||
// json[r'requestedAt'] = null;
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
|
@ -54,6 +70,7 @@ class AuditDeletesResponseDto {
|
|||
? (json[r'ids'] as List).cast<String>()
|
||||
: const [],
|
||||
needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
|
||||
requestedAt: mapDateTime(json, r'requestedAt', ''),
|
||||
);
|
||||
}
|
||||
return null;
|
||||
|
|
|
@ -26,6 +26,11 @@ void main() {
|
|||
// TODO
|
||||
});
|
||||
|
||||
// DateTime requestedAt
|
||||
test('to test the property `requestedAt`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
|
||||
|
|
|
@ -95,8 +95,11 @@ void main() {
|
|||
makeAsset(checksum: "c", remoteId: "1-1"),
|
||||
];
|
||||
expect(db.assets.countSync(), 5);
|
||||
final bool c1 =
|
||||
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
|
||||
final bool c1 = await s.syncRemoteAssetsToDb(
|
||||
owner,
|
||||
_failDiff,
|
||||
(u, n) => remoteAssets,
|
||||
);
|
||||
expect(c1, false);
|
||||
expect(db.assets.countSync(), 5);
|
||||
});
|
||||
|
@ -112,8 +115,11 @@ void main() {
|
|||
makeAsset(checksum: "g", remoteId: "3-1"),
|
||||
];
|
||||
expect(db.assets.countSync(), 5);
|
||||
final bool c1 =
|
||||
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
|
||||
final bool c1 = await s.syncRemoteAssetsToDb(
|
||||
owner,
|
||||
_failDiff,
|
||||
(u, n) => remoteAssets,
|
||||
);
|
||||
expect(c1, true);
|
||||
expect(db.assets.countSync(), 7);
|
||||
});
|
||||
|
@ -129,23 +135,35 @@ void main() {
|
|||
makeAsset(checksum: "j", remoteId: "2-1d"),
|
||||
];
|
||||
expect(db.assets.countSync(), 5);
|
||||
final bool c1 =
|
||||
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
|
||||
final bool c1 = await s.syncRemoteAssetsToDb(
|
||||
owner,
|
||||
_failDiff,
|
||||
(u, n) => remoteAssets,
|
||||
);
|
||||
expect(c1, true);
|
||||
expect(db.assets.countSync(), 8);
|
||||
final bool c2 =
|
||||
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
|
||||
final bool c2 = await s.syncRemoteAssetsToDb(
|
||||
owner,
|
||||
_failDiff,
|
||||
(u, n) => remoteAssets,
|
||||
);
|
||||
expect(c2, false);
|
||||
expect(db.assets.countSync(), 8);
|
||||
remoteAssets.removeAt(4);
|
||||
final bool c3 =
|
||||
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
|
||||
final bool c3 = await s.syncRemoteAssetsToDb(
|
||||
owner,
|
||||
_failDiff,
|
||||
(u, n) => remoteAssets,
|
||||
);
|
||||
expect(c3, true);
|
||||
expect(db.assets.countSync(), 7);
|
||||
remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e"));
|
||||
remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2"));
|
||||
final bool c4 =
|
||||
await s.syncRemoteAssetsToDb(owner, _failDiff, (u) => remoteAssets);
|
||||
final bool c4 = await s.syncRemoteAssetsToDb(
|
||||
owner,
|
||||
_failDiff,
|
||||
(u, n) => remoteAssets,
|
||||
);
|
||||
expect(c4, true);
|
||||
expect(db.assets.countSync(), 9);
|
||||
});
|
||||
|
@ -161,8 +179,8 @@ void main() {
|
|||
final List<String> toDelete = ["2-1", "1-1"];
|
||||
final bool c = await s.syncRemoteAssetsToDb(
|
||||
owner,
|
||||
(user, since) async => (toUpsert, toDelete),
|
||||
(user) => throw Exception(),
|
||||
(user, since) async => (toUpsert, toDelete, null),
|
||||
(user, n) => throw Exception(),
|
||||
);
|
||||
expect(c, true);
|
||||
expect(db.assets.countSync(), 6);
|
||||
|
@ -170,7 +188,10 @@ void main() {
|
|||
});
|
||||
}
|
||||
|
||||
Future<(List<Asset>?, List<String>?)> _failDiff(User user, DateTime time) =>
|
||||
Future.value((null, null));
|
||||
Future<(List<Asset>?, List<String>?, DateTime?)> _failDiff(
|
||||
User user,
|
||||
DateTime time,
|
||||
) =>
|
||||
Future.value((null, null, null));
|
||||
|
||||
class MockHashService extends Mock implements HashService {}
|
||||
|
|
|
@ -6826,6 +6826,10 @@
|
|||
},
|
||||
"needsFullSync": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"requestedAt": {
|
||||
"format": "date-time",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
|
|
@ -30,6 +30,7 @@ export enum PathEntityType {
|
|||
export class AuditDeletesResponseDto {
|
||||
needsFullSync!: boolean;
|
||||
ids!: string[];
|
||||
requestedAt?: Date;
|
||||
}
|
||||
|
||||
export class FileReportDto {
|
||||
|
|
|
@ -61,6 +61,7 @@ describe(AuditService.name, () => {
|
|||
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
|
||||
needsFullSync: true,
|
||||
ids: [],
|
||||
requestedAt: expect.any(Date),
|
||||
});
|
||||
|
||||
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
|
||||
|
@ -77,6 +78,7 @@ describe(AuditService.name, () => {
|
|||
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
|
||||
needsFullSync: false,
|
||||
ids: ['asset-deleted'],
|
||||
requestedAt: expect.any(Date),
|
||||
});
|
||||
|
||||
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
|
||||
|
|
|
@ -4,7 +4,7 @@ import { DateTime } from 'luxon';
|
|||
import { resolve } from 'node:path';
|
||||
import { AccessCore, Permission } from '../access';
|
||||
import { AuthUserDto } from '../auth';
|
||||
import { AUDIT_LOG_MAX_DURATION } from '../domain.constant';
|
||||
import { AUDIT_LOG_CLEANUP_DURATION, AUDIT_LOG_MAX_DURATION } from '../domain.constant';
|
||||
import { usePagination } from '../domain.util';
|
||||
import { JOBS_ASSET_PAGINATION_SIZE } from '../job';
|
||||
import {
|
||||
|
@ -44,7 +44,7 @@ export class AuditService {
|
|||
}
|
||||
|
||||
async handleCleanup(): Promise<boolean> {
|
||||
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate());
|
||||
await this.repository.removeBefore(DateTime.now().minus(AUDIT_LOG_CLEANUP_DURATION).toJSDate());
|
||||
return true;
|
||||
}
|
||||
|
||||
|
@ -52,15 +52,16 @@ export class AuditService {
|
|||
const userId = dto.userId || authUser.id;
|
||||
await this.access.requirePermission(authUser, Permission.TIMELINE_READ, userId);
|
||||
|
||||
const now = DateTime.utc();
|
||||
const duration = now.diff(DateTime.fromJSDate(dto.after));
|
||||
const audits = await this.repository.getAfter(dto.after, {
|
||||
ownerId: userId,
|
||||
entityType: dto.entityType,
|
||||
action: DatabaseAction.DELETE,
|
||||
});
|
||||
|
||||
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.after));
|
||||
|
||||
return {
|
||||
requestedAt: now.toJSDate(),
|
||||
needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
|
||||
ids: audits.map(({ entityId }) => entityId),
|
||||
};
|
||||
|
|
|
@ -4,6 +4,7 @@ import { extname } from 'node:path';
|
|||
import pkg from 'src/../../package.json';
|
||||
|
||||
export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 });
|
||||
export const AUDIT_LOG_CLEANUP_DURATION = Duration.fromObject({ days: 101 });
|
||||
export const ONE_HOUR = Duration.fromObject({ hours: 1 });
|
||||
|
||||
export interface IServerVersion {
|
||||
|
|
6
web/src/api/open-api/api.ts
generated
6
web/src/api/open-api/api.ts
generated
|
@ -959,6 +959,12 @@ export interface AuditDeletesResponseDto {
|
|||
* @memberof AuditDeletesResponseDto
|
||||
*/
|
||||
'needsFullSync': boolean;
|
||||
/**
|
||||
*
|
||||
* @type {string}
|
||||
* @memberof AuditDeletesResponseDto
|
||||
*/
|
||||
'requestedAt'?: string;
|
||||
}
|
||||
/**
|
||||
*
|
||||
|
|
Loading…
Reference in a new issue