feat: postgres reverse geocoding (#5301)
* feat: add system metadata repository for storing key values for internal usage * feat: add database entities for geodata * feat: move reverse geocoding from local-reverse-geocoder to postgresql * infra: disable synchronization for geodata_places table until typeorm supports earth column * feat: remove cities override config as we will default all instances to cities500 now * test: e2e tests don't clear geodata tables on reset
This commit is contained in:
Zack Pollard
GitHub
parent
0108211c0f
commit
698226634e
46 changed files with 368 additions and 645 deletions
24
cli/src/api/open-api/api.ts
generated
24
cli/src/api/open-api/api.ts
generated
|
|
@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto {
|
|||
*/
|
||||
'existingIds': Array<string>;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @export
|
||||
* @enum {string}
|
||||
*/
|
||||
|
||||
export const CitiesFile = {
|
||||
Cities15000: 'cities15000',
|
||||
Cities5000: 'cities5000',
|
||||
Cities1000: 'cities1000',
|
||||
Cities500: 'cities500'
|
||||
} as const;
|
||||
|
||||
export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile];
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @export
|
||||
|
|
@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto {
|
|||
* @interface SystemConfigReverseGeocodingDto
|
||||
*/
|
||||
export interface SystemConfigReverseGeocodingDto {
|
||||
/**
|
||||
*
|
||||
* @type {CitiesFile}
|
||||
* @memberof SystemConfigReverseGeocodingDto
|
||||
*/
|
||||
'citiesFileOverride': CitiesFile;
|
||||
/**
|
||||
*
|
||||
* @type {boolean}
|
||||
|
|
@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto {
|
|||
*/
|
||||
'enabled': boolean;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @export
|
||||
|
|
|
|||
3
mobile/openapi/.openapi-generator/FILES
generated
3
mobile/openapi/.openapi-generator/FILES
generated
|
|
@ -46,7 +46,6 @@ doc/CQMode.md
|
|||
doc/ChangePasswordDto.md
|
||||
doc/CheckExistingAssetsDto.md
|
||||
doc/CheckExistingAssetsResponseDto.md
|
||||
doc/CitiesFile.md
|
||||
doc/ClassificationConfig.md
|
||||
doc/Colorspace.md
|
||||
doc/CreateAlbumDto.md
|
||||
|
|
@ -231,7 +230,6 @@ lib/model/bulk_ids_dto.dart
|
|||
lib/model/change_password_dto.dart
|
||||
lib/model/check_existing_assets_dto.dart
|
||||
lib/model/check_existing_assets_response_dto.dart
|
||||
lib/model/cities_file.dart
|
||||
lib/model/classification_config.dart
|
||||
lib/model/clip_config.dart
|
||||
lib/model/clip_mode.dart
|
||||
|
|
@ -388,7 +386,6 @@ test/bulk_ids_dto_test.dart
|
|||
test/change_password_dto_test.dart
|
||||
test/check_existing_assets_dto_test.dart
|
||||
test/check_existing_assets_response_dto_test.dart
|
||||
test/cities_file_test.dart
|
||||
test/classification_config_test.dart
|
||||
test/clip_config_test.dart
|
||||
test/clip_mode_test.dart
|
||||
|
|
|
|||
1
mobile/openapi/README.md
generated
1
mobile/openapi/README.md
generated
|
|
@ -244,7 +244,6 @@ Class | Method | HTTP request | Description
|
|||
- [ChangePasswordDto](doc//ChangePasswordDto.md)
|
||||
- [CheckExistingAssetsDto](doc//CheckExistingAssetsDto.md)
|
||||
- [CheckExistingAssetsResponseDto](doc//CheckExistingAssetsResponseDto.md)
|
||||
- [CitiesFile](doc//CitiesFile.md)
|
||||
- [ClassificationConfig](doc//ClassificationConfig.md)
|
||||
- [Colorspace](doc//Colorspace.md)
|
||||
- [CreateAlbumDto](doc//CreateAlbumDto.md)
|
||||
|
|
|
|||
14
mobile/openapi/doc/CitiesFile.md
generated
14
mobile/openapi/doc/CitiesFile.md
generated
|
|
@ -1,14 +0,0 @@
|
|||
# openapi.model.CitiesFile
|
||||
|
||||
## Load the model package
|
||||
```dart
|
||||
import 'package:openapi/api.dart';
|
||||
```
|
||||
|
||||
## Properties
|
||||
Name | Type | Description | Notes
|
||||
------------ | ------------- | ------------- | -------------
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
||||
|
||||
|
|
@ -8,7 +8,6 @@ import 'package:openapi/api.dart';
|
|||
## Properties
|
||||
Name | Type | Description | Notes
|
||||
------------ | ------------- | ------------- | -------------
|
||||
**citiesFileOverride** | [**CitiesFile**](CitiesFile.md) | |
|
||||
**enabled** | **bool** | |
|
||||
|
||||
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
|
||||
|
|
|
|||
1
mobile/openapi/lib/api.dart
generated
1
mobile/openapi/lib/api.dart
generated
|
|
@ -83,7 +83,6 @@ part 'model/cq_mode.dart';
|
|||
part 'model/change_password_dto.dart';
|
||||
part 'model/check_existing_assets_dto.dart';
|
||||
part 'model/check_existing_assets_response_dto.dart';
|
||||
part 'model/cities_file.dart';
|
||||
part 'model/classification_config.dart';
|
||||
part 'model/colorspace.dart';
|
||||
part 'model/create_album_dto.dart';
|
||||
|
|
|
|||
2
mobile/openapi/lib/api_client.dart
generated
2
mobile/openapi/lib/api_client.dart
generated
|
|
@ -255,8 +255,6 @@ class ApiClient {
|
|||
return CheckExistingAssetsDto.fromJson(value);
|
||||
case 'CheckExistingAssetsResponseDto':
|
||||
return CheckExistingAssetsResponseDto.fromJson(value);
|
||||
case 'CitiesFile':
|
||||
return CitiesFileTypeTransformer().decode(value);
|
||||
case 'ClassificationConfig':
|
||||
return ClassificationConfig.fromJson(value);
|
||||
case 'Colorspace':
|
||||
|
|
|
|||
3
mobile/openapi/lib/api_helper.dart
generated
3
mobile/openapi/lib/api_helper.dart
generated
|
|
@ -73,9 +73,6 @@ String parameterToString(dynamic value) {
|
|||
if (value is CQMode) {
|
||||
return CQModeTypeTransformer().encode(value).toString();
|
||||
}
|
||||
if (value is CitiesFile) {
|
||||
return CitiesFileTypeTransformer().encode(value).toString();
|
||||
}
|
||||
if (value is Colorspace) {
|
||||
return ColorspaceTypeTransformer().encode(value).toString();
|
||||
}
|
||||
|
|
|
|||
91
mobile/openapi/lib/model/cities_file.dart
generated
91
mobile/openapi/lib/model/cities_file.dart
generated
|
|
@ -1,91 +0,0 @@
|
|||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
part of openapi.api;
|
||||
|
||||
|
||||
class CitiesFile {
|
||||
/// Instantiate a new enum with the provided [value].
|
||||
const CitiesFile._(this.value);
|
||||
|
||||
/// The underlying value of this enum member.
|
||||
final String value;
|
||||
|
||||
@override
|
||||
String toString() => value;
|
||||
|
||||
String toJson() => value;
|
||||
|
||||
static const cities15000 = CitiesFile._(r'cities15000');
|
||||
static const cities5000 = CitiesFile._(r'cities5000');
|
||||
static const cities1000 = CitiesFile._(r'cities1000');
|
||||
static const cities500 = CitiesFile._(r'cities500');
|
||||
|
||||
/// List of all possible values in this [enum][CitiesFile].
|
||||
static const values = <CitiesFile>[
|
||||
cities15000,
|
||||
cities5000,
|
||||
cities1000,
|
||||
cities500,
|
||||
];
|
||||
|
||||
static CitiesFile? fromJson(dynamic value) => CitiesFileTypeTransformer().decode(value);
|
||||
|
||||
static List<CitiesFile>? listFromJson(dynamic json, {bool growable = false,}) {
|
||||
final result = <CitiesFile>[];
|
||||
if (json is List && json.isNotEmpty) {
|
||||
for (final row in json) {
|
||||
final value = CitiesFile.fromJson(row);
|
||||
if (value != null) {
|
||||
result.add(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result.toList(growable: growable);
|
||||
}
|
||||
}
|
||||
|
||||
/// Transformation class that can [encode] an instance of [CitiesFile] to String,
|
||||
/// and [decode] dynamic data back to [CitiesFile].
|
||||
class CitiesFileTypeTransformer {
|
||||
factory CitiesFileTypeTransformer() => _instance ??= const CitiesFileTypeTransformer._();
|
||||
|
||||
const CitiesFileTypeTransformer._();
|
||||
|
||||
String encode(CitiesFile data) => data.value;
|
||||
|
||||
/// Decodes a [dynamic value][data] to a CitiesFile.
|
||||
///
|
||||
/// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully,
|
||||
/// then null is returned. However, if [allowNull] is false and the [dynamic value][data]
|
||||
/// cannot be decoded successfully, then an [UnimplementedError] is thrown.
|
||||
///
|
||||
/// The [allowNull] is very handy when an API changes and a new enum value is added or removed,
|
||||
/// and users are still using an old app with the old code.
|
||||
CitiesFile? decode(dynamic data, {bool allowNull = true}) {
|
||||
if (data != null) {
|
||||
switch (data) {
|
||||
case r'cities15000': return CitiesFile.cities15000;
|
||||
case r'cities5000': return CitiesFile.cities5000;
|
||||
case r'cities1000': return CitiesFile.cities1000;
|
||||
case r'cities500': return CitiesFile.cities500;
|
||||
default:
|
||||
if (!allowNull) {
|
||||
throw ArgumentError('Unknown enum value to decode: $data');
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Singleton [CitiesFileTypeTransformer] instance.
|
||||
static CitiesFileTypeTransformer? _instance;
|
||||
}
|
||||
|
||||
|
|
@ -13,31 +13,25 @@ part of openapi.api;
|
|||
class SystemConfigReverseGeocodingDto {
|
||||
/// Returns a new [SystemConfigReverseGeocodingDto] instance.
|
||||
SystemConfigReverseGeocodingDto({
|
||||
required this.citiesFileOverride,
|
||||
required this.enabled,
|
||||
});
|
||||
|
||||
CitiesFile citiesFileOverride;
|
||||
|
||||
bool enabled;
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) => identical(this, other) || other is SystemConfigReverseGeocodingDto &&
|
||||
other.citiesFileOverride == citiesFileOverride &&
|
||||
other.enabled == enabled;
|
||||
|
||||
@override
|
||||
int get hashCode =>
|
||||
// ignore: unnecessary_parenthesis
|
||||
(citiesFileOverride.hashCode) +
|
||||
(enabled.hashCode);
|
||||
|
||||
@override
|
||||
String toString() => 'SystemConfigReverseGeocodingDto[citiesFileOverride=$citiesFileOverride, enabled=$enabled]';
|
||||
String toString() => 'SystemConfigReverseGeocodingDto[enabled=$enabled]';
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final json = <String, dynamic>{};
|
||||
json[r'citiesFileOverride'] = this.citiesFileOverride;
|
||||
json[r'enabled'] = this.enabled;
|
||||
return json;
|
||||
}
|
||||
|
|
@ -50,7 +44,6 @@ class SystemConfigReverseGeocodingDto {
|
|||
final json = value.cast<String, dynamic>();
|
||||
|
||||
return SystemConfigReverseGeocodingDto(
|
||||
citiesFileOverride: CitiesFile.fromJson(json[r'citiesFileOverride'])!,
|
||||
enabled: mapValueOfType<bool>(json, r'enabled')!,
|
||||
);
|
||||
}
|
||||
|
|
@ -99,7 +92,6 @@ class SystemConfigReverseGeocodingDto {
|
|||
|
||||
/// The list of required keys that must be present in a JSON.
|
||||
static const requiredKeys = <String>{
|
||||
'citiesFileOverride',
|
||||
'enabled',
|
||||
};
|
||||
}
|
||||
|
|
|
|||
21
mobile/openapi/test/cities_file_test.dart
generated
21
mobile/openapi/test/cities_file_test.dart
generated
|
|
@ -1,21 +0,0 @@
|
|||
//
|
||||
// AUTO-GENERATED FILE, DO NOT MODIFY!
|
||||
//
|
||||
// @dart=2.12
|
||||
|
||||
// ignore_for_file: unused_element, unused_import
|
||||
// ignore_for_file: always_put_required_named_parameters_first
|
||||
// ignore_for_file: constant_identifier_names
|
||||
// ignore_for_file: lines_longer_than_80_chars
|
||||
|
||||
import 'package:openapi/api.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
// tests for CitiesFile
|
||||
void main() {
|
||||
|
||||
group('test CitiesFile', () {
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
|
@ -16,11 +16,6 @@ void main() {
|
|||
// final instance = SystemConfigReverseGeocodingDto();
|
||||
|
||||
group('test SystemConfigReverseGeocodingDto', () {
|
||||
// CitiesFile citiesFileOverride
|
||||
test('to test the property `citiesFileOverride`', () async {
|
||||
// TODO
|
||||
});
|
||||
|
||||
// bool enabled
|
||||
test('to test the property `enabled`', () async {
|
||||
// TODO
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ COPY --from=prod /usr/src/app/node_modules ./node_modules
|
|||
COPY --from=prod /usr/src/app/dist ./dist
|
||||
COPY --from=prod /usr/src/app/bin ./bin
|
||||
COPY --from=web /usr/src/app/build ./www
|
||||
COPY server/assets assets
|
||||
COPY server/resources resources
|
||||
COPY server/package.json server/package-lock.json ./
|
||||
COPY server/start*.sh ./
|
||||
RUN npm link && npm cache clean --force
|
||||
|
|
|
|||
|
|
@ -6989,15 +6989,6 @@
|
|||
],
|
||||
"type": "object"
|
||||
},
|
||||
"CitiesFile": {
|
||||
"enum": [
|
||||
"cities15000",
|
||||
"cities5000",
|
||||
"cities1000",
|
||||
"cities500"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ClassificationConfig": {
|
||||
"properties": {
|
||||
"enabled": {
|
||||
|
|
@ -9112,15 +9103,11 @@
|
|||
},
|
||||
"SystemConfigReverseGeocodingDto": {
|
||||
"properties": {
|
||||
"citiesFileOverride": {
|
||||
"$ref": "#/components/schemas/CitiesFile"
|
||||
},
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"citiesFileOverride",
|
||||
"enabled"
|
||||
],
|
||||
"type": "object"
|
||||
|
|
|
|||
264
server/package-lock.json
generated
264
server/package-lock.json
generated
|
|
@ -38,7 +38,6 @@
|
|||
"i18n-iso-countries": "^7.6.0",
|
||||
"ioredis": "^5.3.2",
|
||||
"joi": "^17.10.0",
|
||||
"local-reverse-geocoder": "0.16.5",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.4.2",
|
||||
"mv": "^2.1.1",
|
||||
|
|
@ -4132,18 +4131,6 @@
|
|||
"node": ">=0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/binary": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
|
||||
"integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
|
||||
"dependencies": {
|
||||
"buffers": "~0.1.1",
|
||||
"chainsaw": "~0.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/binary-extensions": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
|
||||
|
|
@ -4329,14 +4316,6 @@
|
|||
"node": ">=4"
|
||||
}
|
||||
},
|
||||
"node_modules/buffers": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
|
||||
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==",
|
||||
"engines": {
|
||||
"node": ">=0.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/buildcheck": {
|
||||
"version": "0.0.6",
|
||||
"resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
|
||||
|
|
@ -4500,17 +4479,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
"node_modules/chainsaw": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
|
||||
"integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
|
||||
"dependencies": {
|
||||
"traverse": ">=0.3.0 <0.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/chalk": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||
|
|
@ -5161,19 +5129,6 @@
|
|||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz",
|
||||
"integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw=="
|
||||
},
|
||||
"node_modules/data-uri-to-buffer": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
|
||||
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
}
|
||||
},
|
||||
"node_modules/date-fns": {
|
||||
"version": "2.30.0",
|
||||
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
|
||||
|
|
@ -6300,28 +6255,6 @@
|
|||
"bser": "2.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/fetch-blob": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
|
||||
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "paypal",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"node-domexception": "^1.0.0",
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20 || >= 14.13"
|
||||
}
|
||||
},
|
||||
"node_modules/figures": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
|
||||
|
|
@ -6575,17 +6508,6 @@
|
|||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/formdata-polyfill": {
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
|
||||
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
|
||||
"dependencies": {
|
||||
"fetch-blob": "^3.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/formidable": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
|
||||
|
|
@ -8323,11 +8245,6 @@
|
|||
"graceful-fs": "^4.1.6"
|
||||
}
|
||||
},
|
||||
"node_modules/kdt": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz",
|
||||
"integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg=="
|
||||
},
|
||||
"node_modules/keyv": {
|
||||
"version": "4.5.3",
|
||||
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz",
|
||||
|
|
@ -8425,41 +8342,6 @@
|
|||
"node": ">=6.11.5"
|
||||
}
|
||||
},
|
||||
"node_modules/local-reverse-geocoder": {
|
||||
"version": "0.16.5",
|
||||
"resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz",
|
||||
"integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==",
|
||||
"hasInstallScript": true,
|
||||
"dependencies": {
|
||||
"async": "^3.2.4",
|
||||
"csv-parse": "^5.5.0",
|
||||
"debug": "^4.3.4",
|
||||
"kdt": "^0.1.0",
|
||||
"node-fetch": "^3.3.2",
|
||||
"unzip-stream": "^0.3.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=11.0.0",
|
||||
"npm": ">=6.4.1"
|
||||
}
|
||||
},
|
||||
"node_modules/local-reverse-geocoder/node_modules/node-fetch": {
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
|
||||
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
|
||||
"dependencies": {
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/node-fetch"
|
||||
}
|
||||
},
|
||||
"node_modules/locate-path": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
|
||||
|
|
@ -9077,24 +8959,6 @@
|
|||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
|
||||
"integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-emoji": {
|
||||
"version": "1.11.0",
|
||||
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
|
||||
|
|
@ -11717,14 +11581,6 @@
|
|||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
||||
},
|
||||
"node_modules/traverse": {
|
||||
"version": "0.3.9",
|
||||
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
|
||||
"integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-kill": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
|
||||
|
|
@ -12314,15 +12170,6 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/unzip-stream": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
|
||||
"integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
|
||||
"dependencies": {
|
||||
"binary": "^0.3.0",
|
||||
"mkdirp": "^0.5.1"
|
||||
}
|
||||
},
|
||||
"node_modules/update-browserslist-db": {
|
||||
"version": "1.0.13",
|
||||
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
|
||||
|
|
@ -12480,14 +12327,6 @@
|
|||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
|
||||
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
|
|
@ -15937,15 +15776,6 @@
|
|||
"integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==",
|
||||
"dev": true
|
||||
},
|
||||
"binary": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
|
||||
"integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
|
||||
"requires": {
|
||||
"buffers": "~0.1.1",
|
||||
"chainsaw": "~0.1.0"
|
||||
}
|
||||
},
|
||||
"binary-extensions": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
|
||||
|
|
@ -16077,11 +15907,6 @@
|
|||
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
|
||||
"integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
|
||||
},
|
||||
"buffers": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
|
||||
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="
|
||||
},
|
||||
"buildcheck": {
|
||||
"version": "0.0.6",
|
||||
"resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
|
||||
|
|
@ -16194,14 +16019,6 @@
|
|||
"integrity": "sha512-UrtAXVcj1mvPBFQ4sKd38daP8dEcXXr5sQe6QNNinaPd0iA/cxg9/l3VrSdL73jgw5sKyuQ6jNgiKO12W3SsVA==",
|
||||
"dev": true
|
||||
},
|
||||
"chainsaw": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
|
||||
"integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
|
||||
"requires": {
|
||||
"traverse": ">=0.3.0 <0.4"
|
||||
}
|
||||
},
|
||||
"chalk": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||
|
|
@ -16681,16 +16498,6 @@
|
|||
"which": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"csv-parse": {
|
||||
"version": "5.5.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz",
|
||||
"integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw=="
|
||||
},
|
||||
"data-uri-to-buffer": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
|
||||
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="
|
||||
},
|
||||
"date-fns": {
|
||||
"version": "2.30.0",
|
||||
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
|
||||
|
|
@ -17523,15 +17330,6 @@
|
|||
"bser": "2.1.1"
|
||||
}
|
||||
},
|
||||
"fetch-blob": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
|
||||
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
|
||||
"requires": {
|
||||
"node-domexception": "^1.0.0",
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"figures": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
|
||||
|
|
@ -17717,14 +17515,6 @@
|
|||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"formdata-polyfill": {
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
|
||||
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
|
||||
"requires": {
|
||||
"fetch-blob": "^3.1.2"
|
||||
}
|
||||
},
|
||||
"formidable": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
|
||||
|
|
@ -19005,11 +18795,6 @@
|
|||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"kdt": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz",
|
||||
"integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg=="
|
||||
},
|
||||
"keyv": {
|
||||
"version": "4.5.3",
|
||||
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz",
|
||||
|
|
@ -19094,31 +18879,6 @@
|
|||
"integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==",
|
||||
"dev": true
|
||||
},
|
||||
"local-reverse-geocoder": {
|
||||
"version": "0.16.5",
|
||||
"resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz",
|
||||
"integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==",
|
||||
"requires": {
|
||||
"async": "^3.2.4",
|
||||
"csv-parse": "^5.5.0",
|
||||
"debug": "^4.3.4",
|
||||
"kdt": "^0.1.0",
|
||||
"node-fetch": "^3.3.2",
|
||||
"unzip-stream": "^0.3.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"node-fetch": {
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
|
||||
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
|
||||
"requires": {
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"locate-path": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
|
||||
|
|
@ -19599,11 +19359,6 @@
|
|||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
|
||||
"integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
|
||||
},
|
||||
"node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="
|
||||
},
|
||||
"node-emoji": {
|
||||
"version": "1.11.0",
|
||||
"resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
|
||||
|
|
@ -21569,11 +21324,6 @@
|
|||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
||||
},
|
||||
"traverse": {
|
||||
"version": "0.3.9",
|
||||
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
|
||||
"integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="
|
||||
},
|
||||
"tree-kill": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
|
||||
|
|
@ -21900,15 +21650,6 @@
|
|||
"integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==",
|
||||
"dev": true
|
||||
},
|
||||
"unzip-stream": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
|
||||
"integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
|
||||
"requires": {
|
||||
"binary": "^0.3.0",
|
||||
"mkdirp": "^0.5.1"
|
||||
}
|
||||
},
|
||||
"update-browserslist-db": {
|
||||
"version": "1.0.13",
|
||||
"resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
|
||||
|
|
@ -22028,11 +21769,6 @@
|
|||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"web-streams-polyfill": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
|
||||
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
|
||||
},
|
||||
"webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.22.11",
|
||||
"@immich/cli": "^2.0.3",
|
||||
"@nestjs/bullmq": "^10.0.1",
|
||||
"@nestjs/common": "^10.2.2",
|
||||
"@nestjs/config": "^3.0.0",
|
||||
|
|
@ -65,10 +66,8 @@
|
|||
"glob": "^10.3.3",
|
||||
"handlebars": "^4.7.8",
|
||||
"i18n-iso-countries": "^7.6.0",
|
||||
"@immich/cli": "^2.0.3",
|
||||
"ioredis": "^5.3.2",
|
||||
"joi": "^17.10.0",
|
||||
"local-reverse-geocoder": "0.16.5",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.4.2",
|
||||
"mv": "^2.1.1",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { AssetType, CitiesFile, ExifEntity, SystemConfigKey } from '@app/infra/entities';
|
||||
import { AssetType, ExifEntity, SystemConfigKey } from '@app/infra/entities';
|
||||
import {
|
||||
assetStub,
|
||||
newAlbumRepositoryMock,
|
||||
|
|
@ -15,7 +15,7 @@ import { randomBytes } from 'crypto';
|
|||
import { Stats } from 'fs';
|
||||
import { constants } from 'fs/promises';
|
||||
import { when } from 'jest-when';
|
||||
import { JobName, QueueName } from '../job';
|
||||
import { JobName } from '../job';
|
||||
import {
|
||||
IAlbumRepository,
|
||||
IAssetRepository,
|
||||
|
|
@ -78,10 +78,7 @@ describe(MetadataService.name, () => {
|
|||
|
||||
describe('init', () => {
|
||||
beforeEach(async () => {
|
||||
configMock.load.mockResolvedValue([
|
||||
{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true },
|
||||
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_500 },
|
||||
]);
|
||||
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]);
|
||||
|
||||
await sut.init();
|
||||
});
|
||||
|
|
@ -90,42 +87,10 @@ describe(MetadataService.name, () => {
|
|||
configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]);
|
||||
|
||||
await sut.init();
|
||||
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
|
||||
expect(jobMock.pause).toHaveBeenCalledTimes(1);
|
||||
expect(metadataMock.init).toHaveBeenCalledTimes(1);
|
||||
expect(jobMock.resume).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should return if deleteCache is false and the cities precision has not changed', async () => {
|
||||
await sut.init();
|
||||
|
||||
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
|
||||
expect(jobMock.pause).toHaveBeenCalledTimes(1);
|
||||
expect(metadataMock.init).toHaveBeenCalledTimes(1);
|
||||
expect(jobMock.resume).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should re-init if deleteCache is false but the cities precision has changed', async () => {
|
||||
configMock.load.mockResolvedValue([
|
||||
{ key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_1000 },
|
||||
]);
|
||||
|
||||
await sut.init();
|
||||
|
||||
expect(metadataMock.deleteCache).not.toHaveBeenCalled();
|
||||
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
|
||||
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_1000 });
|
||||
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
|
||||
});
|
||||
|
||||
it('should re-init and delete cache if deleteCache is true', async () => {
|
||||
await sut.init(true);
|
||||
|
||||
expect(metadataMock.deleteCache).toHaveBeenCalled();
|
||||
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
|
||||
expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_500 });
|
||||
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handleLivePhotoLinking', () => {
|
||||
|
|
|
|||
|
|
@ -97,31 +97,24 @@ export class MetadataService {
|
|||
this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
|
||||
}
|
||||
|
||||
async init(deleteCache = false) {
|
||||
async init() {
|
||||
if (!this.subscription) {
|
||||
this.subscription = this.configCore.config$.subscribe(() => this.init());
|
||||
}
|
||||
|
||||
const { reverseGeocoding } = await this.configCore.getConfig();
|
||||
const { citiesFileOverride } = reverseGeocoding;
|
||||
const { enabled } = reverseGeocoding;
|
||||
|
||||
if (!reverseGeocoding.enabled) {
|
||||
if (!enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (deleteCache) {
|
||||
await this.repository.deleteCache();
|
||||
} else if (this.oldCities && this.oldCities === citiesFileOverride) {
|
||||
return;
|
||||
}
|
||||
|
||||
await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
|
||||
await this.repository.init({ citiesFileOverride });
|
||||
await this.repository.init();
|
||||
await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
|
||||
|
||||
this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`);
|
||||
this.oldCities = citiesFileOverride;
|
||||
this.logger.log(`Initialized local reverse geocoder`);
|
||||
} catch (error: Error | any) {
|
||||
this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack);
|
||||
}
|
||||
|
|
@ -258,8 +251,9 @@ export class MetadataService {
|
|||
}
|
||||
|
||||
try {
|
||||
const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude });
|
||||
Object.assign(exifData, { city, state, country });
|
||||
const reverseGeocode = await this.repository.reverseGeocode({ latitude, longitude });
|
||||
if (!reverseGeocode) return;
|
||||
Object.assign(exifData, reverseGeocode);
|
||||
} catch (error: Error | any) {
|
||||
this.logger.warn(
|
||||
`Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ export * from './shared-link.repository';
|
|||
export * from './smart-info.repository';
|
||||
export * from './storage.repository';
|
||||
export * from './system-config.repository';
|
||||
export * from './system-metadata.repository';
|
||||
export * from './tag.repository';
|
||||
export * from './user-token.repository';
|
||||
export * from './user.repository';
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import { Tags } from 'exiftool-vendored';
|
||||
import { InitOptions } from 'local-reverse-geocoder';
|
||||
|
||||
export const IMetadataRepository = 'IMetadataRepository';
|
||||
|
||||
|
|
@ -31,9 +30,8 @@ export interface ImmichTags extends Omit<Tags, 'FocalLength' | 'Duration'> {
|
|||
}
|
||||
|
||||
export interface IMetadataRepository {
|
||||
init(options: Partial<InitOptions>): Promise<void>;
|
||||
init(): Promise<void>;
|
||||
teardown(): Promise<void>;
|
||||
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
|
||||
deleteCache(): Promise<void>;
|
||||
reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
|
||||
getExifTags(path: string): Promise<ImmichTags | null>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
import { SystemMetadata } from '@app/infra/entities';
|
||||
|
||||
export const ISystemMetadataRepository = 'ISystemMetadataRepository';
|
||||
|
||||
export interface ISystemMetadataRepository {
|
||||
get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null>;
|
||||
set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void>;
|
||||
}
|
||||
|
|
@ -1,12 +1,6 @@
|
|||
import { CitiesFile } from '@app/infra/entities';
|
||||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { IsBoolean, IsEnum } from 'class-validator';
|
||||
import { IsBoolean } from 'class-validator';
|
||||
|
||||
export class SystemConfigReverseGeocodingDto {
|
||||
@IsBoolean()
|
||||
enabled!: boolean;
|
||||
|
||||
@IsEnum(CitiesFile)
|
||||
@ApiProperty({ enum: CitiesFile, enumName: 'CitiesFile' })
|
||||
citiesFileOverride!: CitiesFile;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import {
|
||||
AudioCodec,
|
||||
CitiesFile,
|
||||
Colorspace,
|
||||
CQMode,
|
||||
SystemConfig,
|
||||
|
|
@ -85,7 +84,6 @@ export const defaults = Object.freeze<SystemConfig>({
|
|||
},
|
||||
reverseGeocoding: {
|
||||
enabled: true,
|
||||
citiesFileOverride: CitiesFile.CITIES_500,
|
||||
},
|
||||
oauth: {
|
||||
enabled: false,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import {
|
||||
AudioCodec,
|
||||
CitiesFile,
|
||||
Colorspace,
|
||||
CQMode,
|
||||
SystemConfig,
|
||||
|
|
@ -85,7 +84,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
|
|||
},
|
||||
reverseGeocoding: {
|
||||
enabled: true,
|
||||
citiesFileOverride: CitiesFile.CITIES_500,
|
||||
},
|
||||
oauth: {
|
||||
autoLaunch: true,
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ export class SystemConfigService {
|
|||
return this.repository.fetchStyle(styleUrl);
|
||||
}
|
||||
|
||||
return JSON.parse(await this.repository.readFile(`./assets/style-${theme}.json`));
|
||||
return JSON.parse(await this.repository.readFile(`./resources/style-${theme}.json`));
|
||||
}
|
||||
|
||||
async getCustomCss(): Promise<string> {
|
||||
|
|
|
|||
10
server/src/infra/entities/geodata-admin1.entity.ts
Normal file
10
server/src/infra/entities/geodata-admin1.entity.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { Column, Entity, PrimaryColumn } from 'typeorm';
|
||||
|
||||
@Entity('geodata_admin1')
|
||||
export class GeodataAdmin1Entity {
|
||||
@PrimaryColumn({ type: 'varchar' })
|
||||
key!: string;
|
||||
|
||||
@Column({ type: 'varchar' })
|
||||
name!: string;
|
||||
}
|
||||
10
server/src/infra/entities/geodata-admin2.entity.ts
Normal file
10
server/src/infra/entities/geodata-admin2.entity.ts
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import { Column, Entity, PrimaryColumn } from 'typeorm';
|
||||
|
||||
@Entity('geodata_admin2')
|
||||
export class GeodataAdmin2Entity {
|
||||
@PrimaryColumn({ type: 'varchar' })
|
||||
key!: string;
|
||||
|
||||
@Column({ type: 'varchar' })
|
||||
name!: string;
|
||||
}
|
||||
59
server/src/infra/entities/geodata-places.entity.ts
Normal file
59
server/src/infra/entities/geodata-places.entity.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { GeodataAdmin1Entity } from '@app/infra/entities/geodata-admin1.entity';
|
||||
import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity';
|
||||
import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm';
|
||||
|
||||
@Entity('geodata_places', { synchronize: false })
|
||||
export class GeodataPlacesEntity {
|
||||
@PrimaryColumn({ type: 'integer' })
|
||||
id!: number;
|
||||
|
||||
@Column({ type: 'varchar', length: 200 })
|
||||
name!: string;
|
||||
|
||||
@Column({ type: 'float' })
|
||||
longitude!: number;
|
||||
|
||||
@Column({ type: 'float' })
|
||||
latitude!: number;
|
||||
|
||||
// @Column({
|
||||
// generatedType: 'STORED',
|
||||
// asExpression: 'll_to_earth((latitude)::double precision, (longitude)::double precision)',
|
||||
// type: 'earth',
|
||||
// })
|
||||
earthCoord!: unknown;
|
||||
|
||||
@Column({ type: 'char', length: 2 })
|
||||
countryCode!: string;
|
||||
|
||||
@Column({ type: 'varchar', length: 20, nullable: true })
|
||||
admin1Code!: string;
|
||||
|
||||
@Column({ type: 'varchar', length: 80, nullable: true })
|
||||
admin2Code!: string;
|
||||
|
||||
@Column({
|
||||
type: 'varchar',
|
||||
generatedType: 'STORED',
|
||||
asExpression: `"countryCode" || '.' || "admin1Code"`,
|
||||
nullable: true,
|
||||
})
|
||||
admin1Key!: string;
|
||||
|
||||
@ManyToOne(() => GeodataAdmin1Entity, { eager: true, nullable: true, createForeignKeyConstraints: false })
|
||||
admin1!: GeodataAdmin1Entity;
|
||||
|
||||
@Column({
|
||||
type: 'varchar',
|
||||
generatedType: 'STORED',
|
||||
asExpression: `"countryCode" || '.' || "admin1Code" || '.' || "admin2Code"`,
|
||||
nullable: true,
|
||||
})
|
||||
admin2Key!: string;
|
||||
|
||||
@ManyToOne(() => GeodataAdmin2Entity, { eager: true, nullable: true, createForeignKeyConstraints: false })
|
||||
admin2!: GeodataAdmin2Entity;
|
||||
|
||||
@Column({ type: 'date' })
|
||||
modificationDate!: Date;
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity';
|
||||
import { ActivityEntity } from './activity.entity';
|
||||
import { AlbumEntity } from './album.entity';
|
||||
import { APIKeyEntity } from './api-key.entity';
|
||||
|
|
@ -6,6 +7,8 @@ import { AssetJobStatusEntity } from './asset-job-status.entity';
|
|||
import { AssetEntity } from './asset.entity';
|
||||
import { AuditEntity } from './audit.entity';
|
||||
import { ExifEntity } from './exif.entity';
|
||||
import { GeodataAdmin1Entity } from './geodata-admin1.entity';
|
||||
import { GeodataPlacesEntity } from './geodata-places.entity';
|
||||
import { LibraryEntity } from './library.entity';
|
||||
import { MoveEntity } from './move.entity';
|
||||
import { PartnerEntity } from './partner.entity';
|
||||
|
|
@ -13,6 +16,7 @@ import { PersonEntity } from './person.entity';
|
|||
import { SharedLinkEntity } from './shared-link.entity';
|
||||
import { SmartInfoEntity } from './smart-info.entity';
|
||||
import { SystemConfigEntity } from './system-config.entity';
|
||||
import { SystemMetadataEntity } from './system-metadata.entity';
|
||||
import { TagEntity } from './tag.entity';
|
||||
import { UserTokenEntity } from './user-token.entity';
|
||||
import { UserEntity } from './user.entity';
|
||||
|
|
@ -25,6 +29,9 @@ export * from './asset-job-status.entity';
|
|||
export * from './asset.entity';
|
||||
export * from './audit.entity';
|
||||
export * from './exif.entity';
|
||||
export * from './geodata-admin1.entity';
|
||||
export * from './geodata-admin2.entity';
|
||||
export * from './geodata-places.entity';
|
||||
export * from './library.entity';
|
||||
export * from './move.entity';
|
||||
export * from './partner.entity';
|
||||
|
|
@ -32,6 +39,7 @@ export * from './person.entity';
|
|||
export * from './shared-link.entity';
|
||||
export * from './smart-info.entity';
|
||||
export * from './system-config.entity';
|
||||
export * from './system-metadata.entity';
|
||||
export * from './tag.entity';
|
||||
export * from './user-token.entity';
|
||||
export * from './user.entity';
|
||||
|
|
@ -45,12 +53,16 @@ export const databaseEntities = [
|
|||
AssetJobStatusEntity,
|
||||
AuditEntity,
|
||||
ExifEntity,
|
||||
GeodataPlacesEntity,
|
||||
GeodataAdmin1Entity,
|
||||
GeodataAdmin2Entity,
|
||||
MoveEntity,
|
||||
PartnerEntity,
|
||||
PersonEntity,
|
||||
SharedLinkEntity,
|
||||
SmartInfoEntity,
|
||||
SystemConfigEntity,
|
||||
SystemMetadataEntity,
|
||||
TagEntity,
|
||||
UserEntity,
|
||||
UserTokenEntity,
|
||||
|
|
|
|||
|
|
@ -66,7 +66,6 @@ export enum SystemConfigKey {
|
|||
MAP_DARK_STYLE = 'map.darkStyle',
|
||||
|
||||
REVERSE_GEOCODING_ENABLED = 'reverseGeocoding.enabled',
|
||||
REVERSE_GEOCODING_CITIES_FILE_OVERRIDE = 'reverseGeocoding.citiesFileOverride',
|
||||
|
||||
NEW_VERSION_CHECK_ENABLED = 'newVersionCheck.enabled',
|
||||
|
||||
|
|
@ -145,13 +144,6 @@ export enum Colorspace {
|
|||
P3 = 'p3',
|
||||
}
|
||||
|
||||
export enum CitiesFile {
|
||||
CITIES_15000 = 'cities15000',
|
||||
CITIES_5000 = 'cities5000',
|
||||
CITIES_1000 = 'cities1000',
|
||||
CITIES_500 = 'cities500',
|
||||
}
|
||||
|
||||
export interface SystemConfig {
|
||||
ffmpeg: {
|
||||
crf: number;
|
||||
|
|
@ -200,7 +192,6 @@ export interface SystemConfig {
|
|||
};
|
||||
reverseGeocoding: {
|
||||
enabled: boolean;
|
||||
citiesFileOverride: CitiesFile;
|
||||
};
|
||||
oauth: {
|
||||
enabled: boolean;
|
||||
|
|
|
|||
18
server/src/infra/entities/system-metadata.entity.ts
Normal file
18
server/src/infra/entities/system-metadata.entity.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import { Column, Entity, PrimaryColumn } from 'typeorm';
|
||||
|
||||
@Entity('system_metadata')
|
||||
export class SystemMetadataEntity {
|
||||
@PrimaryColumn()
|
||||
key!: string;
|
||||
|
||||
@Column({ type: 'jsonb', default: '{}', transformer: { to: JSON.stringify, from: JSON.parse } })
|
||||
value!: { [key: string]: unknown };
|
||||
}
|
||||
|
||||
export enum SystemMetadataKey {
|
||||
REVERSE_GEOCODING_STATE = 'reverse-geocoding-state',
|
||||
}
|
||||
|
||||
export interface SystemMetadata extends Record<SystemMetadataKey, { [key: string]: unknown }> {
|
||||
[SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
|
||||
}
|
||||
|
|
@ -74,6 +74,3 @@ function parseTypeSenseConfig(): ConfigurationOptions {
|
|||
}
|
||||
|
||||
export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig();
|
||||
|
||||
export const REVERSE_GEOCODING_DUMP_DIRECTORY =
|
||||
process.env.REVERSE_GEOCODING_DUMP_DIRECTORY || process.cwd() + '/.reverse-geocoding-dump/';
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import {
|
|||
ISmartInfoRepository,
|
||||
IStorageRepository,
|
||||
ISystemConfigRepository,
|
||||
ISystemMetadataRepository,
|
||||
ITagRepository,
|
||||
IUserRepository,
|
||||
IUserTokenRepository,
|
||||
|
|
@ -56,6 +57,7 @@ import {
|
|||
SharedLinkRepository,
|
||||
SmartInfoRepository,
|
||||
SystemConfigRepository,
|
||||
SystemMetadataRepository,
|
||||
TagRepository,
|
||||
TypesenseRepository,
|
||||
UserRepository,
|
||||
|
|
@ -84,6 +86,7 @@ const providers: Provider[] = [
|
|||
{ provide: ISmartInfoRepository, useClass: SmartInfoRepository },
|
||||
{ provide: IStorageRepository, useClass: FilesystemProvider },
|
||||
{ provide: ISystemConfigRepository, useClass: SystemConfigRepository },
|
||||
{ provide: ISystemMetadataRepository, useClass: SystemMetadataRepository },
|
||||
{ provide: ITagRepository, useClass: TagRepository },
|
||||
{ provide: IMediaRepository, useClass: MediaRepository },
|
||||
{ provide: IUserRepository, useClass: UserRepository },
|
||||
|
|
|
|||
14
server/src/infra/migrations/1700345818045-SystemMetadata.ts
Normal file
14
server/src/infra/migrations/1700345818045-SystemMetadata.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
import { MigrationInterface, QueryRunner } from "typeorm";
|
||||
|
||||
export class SystemMetadata1700345818045 implements MigrationInterface {
|
||||
name = 'SystemMetadata1700345818045'
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`CREATE TABLE "system_metadata" ("key" character varying NOT NULL, "value" jsonb NOT NULL DEFAULT '{}', CONSTRAINT "PK_fa94f6857470fb5b81ec6084465" PRIMARY KEY ("key"))`);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DROP TABLE "system_metadata"`);
|
||||
}
|
||||
|
||||
}
|
||||
29
server/src/infra/migrations/1700362016675-Geodata.ts
Normal file
29
server/src/infra/migrations/1700362016675-Geodata.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import { MigrationInterface, QueryRunner } from "typeorm";
|
||||
|
||||
export class Geodata1700362016675 implements MigrationInterface {
|
||||
name = 'Geodata1700362016675'
|
||||
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS cube`)
|
||||
await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS earthdistance`)
|
||||
await queryRunner.query(`CREATE TABLE "geodata_admin2" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_1e3886455dbb684d6f6b4756726" PRIMARY KEY ("key"))`);
|
||||
await queryRunner.query(`CREATE TABLE "geodata_admin1" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_3fe3a89c5aac789d365871cb172" PRIMARY KEY ("key"))`);
|
||||
await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin1Key","\"countryCode\" || '.' || \"admin1Code\""]);
|
||||
await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin2Key","\"countryCode\" || '.' || \"admin1Code\" || '.' || \"admin2Code\""]);
|
||||
await queryRunner.query(`CREATE TABLE "geodata_places" ("id" integer NOT NULL, "name" character varying(200) NOT NULL, "longitude" double precision NOT NULL, "latitude" double precision NOT NULL, "countryCode" character(2) NOT NULL, "admin1Code" character varying(20), "admin2Code" character varying(80), "admin1Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code") STORED, "admin2Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code" || '.' || "admin2Code") STORED, "modificationDate" date NOT NULL, CONSTRAINT "PK_c29918988912ef4036f3d7fbff4" PRIMARY KEY ("id"))`);
|
||||
await queryRunner.query(`ALTER TABLE "geodata_places" ADD "earthCoord" earth GENERATED ALWAYS AS (ll_to_earth(latitude, longitude)) STORED`)
|
||||
await queryRunner.query(`CREATE INDEX "IDX_geodata_gist_earthcoord" ON "geodata_places" USING gist ("earthCoord");`)
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.query(`DROP INDEX "IDX_geodata_gist_earthcoord"`);
|
||||
await queryRunner.query(`DROP TABLE "geodata_places"`);
|
||||
await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin2Key","immich","public","geodata_places"]);
|
||||
await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin1Key","immich","public","geodata_places"]);
|
||||
await queryRunner.query(`DROP TABLE "geodata_admin1"`);
|
||||
await queryRunner.query(`DROP TABLE "geodata_admin2"`);
|
||||
await queryRunner.query(`DROP EXTENSION cube`);
|
||||
await queryRunner.query(`DROP EXTENSION earthdistance`);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ export * from './server-info.repository';
|
|||
export * from './shared-link.repository';
|
||||
export * from './smart-info.repository';
|
||||
export * from './system-config.repository';
|
||||
export * from './system-metadata.repository';
|
||||
export * from './tag.repository';
|
||||
export * from './typesense.repository';
|
||||
export * from './user-token.repository';
|
||||
|
|
|
|||
|
|
@ -1,77 +1,182 @@
|
|||
import { GeoPoint, IMetadataRepository, ImmichTags, ReverseGeocodeResult } from '@app/domain';
|
||||
import { REVERSE_GEOCODING_DUMP_DIRECTORY } from '@app/infra';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import {
|
||||
GeoPoint,
|
||||
IMetadataRepository,
|
||||
ImmichTags,
|
||||
ISystemMetadataRepository,
|
||||
ReverseGeocodeResult,
|
||||
} from '@app/domain';
|
||||
import { GeodataAdmin1Entity, GeodataAdmin2Entity, GeodataPlacesEntity, SystemMetadataKey } from '@app/infra/entities';
|
||||
import { DatabaseLock } from '@app/infra/utils/database-locks';
|
||||
import { Inject, Logger } from '@nestjs/common';
|
||||
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
|
||||
import { DefaultReadTaskOptions, exiftool } from 'exiftool-vendored';
|
||||
import { readdir, rm } from 'fs/promises';
|
||||
import { createReadStream, existsSync } from 'fs';
|
||||
import { readFile } from 'fs/promises';
|
||||
import * as geotz from 'geo-tz';
|
||||
import { getName } from 'i18n-iso-countries';
|
||||
import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder';
|
||||
import path from 'path';
|
||||
import { promisify } from 'util';
|
||||
import * as readLine from 'readline';
|
||||
import { DataSource, DeepPartial, QueryRunner, Repository } from 'typeorm';
|
||||
|
||||
export interface AdminCode {
|
||||
name: string;
|
||||
asciiName: string;
|
||||
geoNameId: string;
|
||||
}
|
||||
type GeoEntity = GeodataPlacesEntity | GeodataAdmin1Entity | GeodataAdmin2Entity;
|
||||
type GeoEntityClass = typeof GeodataPlacesEntity | typeof GeodataAdmin1Entity | typeof GeodataAdmin2Entity;
|
||||
|
||||
export type GeoData = AddressObject & {
|
||||
admin1Code?: AdminCode | string;
|
||||
admin2Code?: AdminCode | string;
|
||||
};
|
||||
const CITIES_FILE = 'cities500.txt';
|
||||
|
||||
const lookup = promisify<GeoPoint[], number, AddressObject[][]>(geocoder.lookUp).bind(geocoder);
|
||||
|
||||
@Injectable()
|
||||
export class MetadataRepository implements IMetadataRepository {
|
||||
constructor(
|
||||
@InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository<GeodataPlacesEntity>,
|
||||
@InjectRepository(GeodataAdmin1Entity) private readonly geodataAdmin1Repository: Repository<GeodataAdmin1Entity>,
|
||||
@InjectRepository(GeodataAdmin2Entity) private readonly geodataAdmin2Repository: Repository<GeodataAdmin2Entity>,
|
||||
@Inject(ISystemMetadataRepository) private readonly systemMetadataRepository: ISystemMetadataRepository,
|
||||
@InjectDataSource() private dataSource: DataSource,
|
||||
) {}
|
||||
|
||||
private logger = new Logger(MetadataRepository.name);
|
||||
|
||||
async init(options: Partial<InitOptions>): Promise<void> {
|
||||
return new Promise<void>((resolve) => {
|
||||
geocoder.init(
|
||||
{
|
||||
load: {
|
||||
admin1: true,
|
||||
admin2: true,
|
||||
admin3And4: false,
|
||||
alternateNames: false,
|
||||
},
|
||||
countries: [],
|
||||
dumpDirectory: REVERSE_GEOCODING_DUMP_DIRECTORY,
|
||||
...options,
|
||||
},
|
||||
resolve,
|
||||
);
|
||||
async init(): Promise<void> {
|
||||
this.logger.log('Initializing metadata repository');
|
||||
const geodataDate = await readFile('/usr/src/resources/geodata-date.txt', 'utf8');
|
||||
|
||||
await this.geodataPlacesRepository.query('SELECT pg_advisory_lock($1)', [DatabaseLock.GeodataImport]);
|
||||
|
||||
const geocodingMetadata = await this.systemMetadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
|
||||
|
||||
if (geocodingMetadata?.lastUpdate === geodataDate) {
|
||||
await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
|
||||
return;
|
||||
}
|
||||
|
||||
this.logger.log('Importing geodata to database from file');
|
||||
|
||||
const queryRunner = this.dataSource.createQueryRunner();
|
||||
await queryRunner.connect();
|
||||
|
||||
try {
|
||||
await queryRunner.startTransaction();
|
||||
|
||||
await this.loadCities500(queryRunner);
|
||||
await this.loadAdmin1(queryRunner);
|
||||
await this.loadAdmin2(queryRunner);
|
||||
|
||||
await queryRunner.commitTransaction();
|
||||
} catch (e) {
|
||||
this.logger.fatal('Error importing geodata', e);
|
||||
await queryRunner.rollbackTransaction();
|
||||
throw e;
|
||||
} finally {
|
||||
await queryRunner.release();
|
||||
}
|
||||
|
||||
await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
|
||||
lastUpdate: geodataDate,
|
||||
lastImportFileName: CITIES_FILE,
|
||||
});
|
||||
|
||||
await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
|
||||
this.logger.log('Geodata import completed');
|
||||
}
|
||||
|
||||
private async loadGeodataToTableFromFile<T extends GeoEntity>(
|
||||
queryRunner: QueryRunner,
|
||||
lineToEntityMapper: (lineSplit: string[]) => T,
|
||||
filePath: string,
|
||||
entity: GeoEntityClass,
|
||||
) {
|
||||
if (!existsSync(filePath)) {
|
||||
this.logger.error(`Geodata file ${filePath} not found`);
|
||||
throw new Error(`Geodata file ${filePath} not found`);
|
||||
}
|
||||
await queryRunner.manager.clear(entity);
|
||||
|
||||
const input = createReadStream(filePath);
|
||||
let buffer: DeepPartial<T>[] = [];
|
||||
const lineReader = readLine.createInterface({ input: input });
|
||||
|
||||
for await (const line of lineReader) {
|
||||
const lineSplit = line.split('\t');
|
||||
buffer.push(lineToEntityMapper(lineSplit));
|
||||
if (buffer.length > 1000) {
|
||||
await queryRunner.manager.save(buffer);
|
||||
buffer = [];
|
||||
}
|
||||
}
|
||||
await queryRunner.manager.save(buffer);
|
||||
}
|
||||
|
||||
private async loadCities500(queryRunner: QueryRunner) {
|
||||
await this.loadGeodataToTableFromFile<GeodataPlacesEntity>(
|
||||
queryRunner,
|
||||
(lineSplit: string[]) =>
|
||||
this.geodataPlacesRepository.create({
|
||||
id: parseInt(lineSplit[0]),
|
||||
name: lineSplit[1],
|
||||
latitude: parseFloat(lineSplit[4]),
|
||||
longitude: parseFloat(lineSplit[5]),
|
||||
countryCode: lineSplit[8],
|
||||
admin1Code: lineSplit[10],
|
||||
admin2Code: lineSplit[11],
|
||||
modificationDate: lineSplit[18],
|
||||
}),
|
||||
`/usr/src/resources/${CITIES_FILE}`,
|
||||
GeodataPlacesEntity,
|
||||
);
|
||||
}
|
||||
|
||||
private async loadAdmin1(queryRunner: QueryRunner) {
|
||||
await this.loadGeodataToTableFromFile<GeodataAdmin1Entity>(
|
||||
queryRunner,
|
||||
(lineSplit: string[]) =>
|
||||
this.geodataAdmin1Repository.create({
|
||||
key: lineSplit[0],
|
||||
name: lineSplit[1],
|
||||
}),
|
||||
'/usr/src/resources/admin1CodesASCII.txt',
|
||||
GeodataAdmin1Entity,
|
||||
);
|
||||
}
|
||||
|
||||
private async loadAdmin2(queryRunner: QueryRunner) {
|
||||
await this.loadGeodataToTableFromFile<GeodataAdmin2Entity>(
|
||||
queryRunner,
|
||||
(lineSplit: string[]) =>
|
||||
this.geodataAdmin2Repository.create({
|
||||
key: lineSplit[0],
|
||||
name: lineSplit[1],
|
||||
}),
|
||||
'/usr/src/resources/admin2Codes.txt',
|
||||
GeodataAdmin2Entity,
|
||||
);
|
||||
}
|
||||
|
||||
async teardown() {
|
||||
await exiftool.end();
|
||||
}
|
||||
|
||||
async deleteCache() {
|
||||
const dumpDirectory = REVERSE_GEOCODING_DUMP_DIRECTORY;
|
||||
if (dumpDirectory) {
|
||||
// delete contents
|
||||
const items = await readdir(dumpDirectory, { withFileTypes: true });
|
||||
const folders = items.filter((item) => item.isDirectory());
|
||||
for (const { name } of folders) {
|
||||
await rm(path.join(dumpDirectory, name), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
|
||||
async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
|
||||
this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
|
||||
|
||||
const [address] = await lookup([point], 1);
|
||||
this.logger.verbose(`Raw: ${JSON.stringify(address, null, 2)}`);
|
||||
const response = await this.geodataPlacesRepository
|
||||
.createQueryBuilder('geoplaces')
|
||||
.leftJoinAndSelect('geoplaces.admin1', 'admin1')
|
||||
.leftJoinAndSelect('geoplaces.admin2', 'admin2')
|
||||
.where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
|
||||
.orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
|
||||
.limit(1)
|
||||
.getOne();
|
||||
|
||||
const { countryCode, name: city, admin1Code, admin2Code } = address[0] as GeoData;
|
||||
if (!response) {
|
||||
this.logger.warn(
|
||||
`Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
|
||||
|
||||
const { countryCode, name: city, admin1, admin2 } = response;
|
||||
const country = getName(countryCode, 'en') ?? null;
|
||||
const stateParts = [(admin2Code as AdminCode)?.name, (admin1Code as AdminCode)?.name].filter((name) => !!name);
|
||||
const stateParts = [admin2?.name, admin1?.name].filter((name) => !!name);
|
||||
const state = stateParts.length > 0 ? stateParts.join(', ') : null;
|
||||
this.logger.debug(`Normalized: ${JSON.stringify({ country, state, city })}`);
|
||||
|
||||
return { country, state, city };
|
||||
}
|
||||
|
|
|
|||
20
server/src/infra/repositories/system-metadata.repository.ts
Normal file
20
server/src/infra/repositories/system-metadata.repository.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { ISystemMetadataRepository } from '@app/domain/repositories/system-metadata.repository';
|
||||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import { Repository } from 'typeorm';
|
||||
import { SystemMetadata, SystemMetadataEntity } from '../entities';
|
||||
|
||||
export class SystemMetadataRepository implements ISystemMetadataRepository {
|
||||
constructor(
|
||||
@InjectRepository(SystemMetadataEntity)
|
||||
private repository: Repository<SystemMetadataEntity>,
|
||||
) {}
|
||||
async get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null> {
|
||||
const metadata = await this.repository.findOne({ where: { key } });
|
||||
if (!metadata) return null;
|
||||
return metadata.value as SystemMetadata[T];
|
||||
}
|
||||
|
||||
async set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void> {
|
||||
await this.repository.upsert({ key, value }, { conflictPaths: { key: true } });
|
||||
}
|
||||
}
|
||||
3
server/src/infra/utils/database-locks.ts
Normal file
3
server/src/infra/utils/database-locks.ts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
export enum DatabaseLock {
|
||||
GeodataImport = 100,
|
||||
}
|
||||
|
|
@ -92,16 +92,6 @@ export class AppService {
|
|||
[JobName.LIBRARY_QUEUE_CLEANUP]: () => this.libraryService.handleQueueCleanup(),
|
||||
});
|
||||
|
||||
process.on('uncaughtException', async (error: Error | any) => {
|
||||
const isCsvError = error.code === 'CSV_RECORD_INCONSISTENT_FIELDS_LENGTH';
|
||||
if (!isCsvError) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
this.logger.warn('Geocoding csv parse error, trying again without cache...');
|
||||
await this.metadataService.init(true);
|
||||
});
|
||||
|
||||
await this.metadataService.init();
|
||||
await this.searchService.init();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ import { IMetadataRepository } from '@app/domain';
|
|||
|
||||
export const newMetadataRepositoryMock = (): jest.Mocked<IMetadataRepository> => {
|
||||
return {
|
||||
deleteCache: jest.fn(),
|
||||
getExifTags: jest.fn(),
|
||||
init: jest.fn(),
|
||||
teardown: jest.fn(),
|
||||
|
|
|
|||
|
|
@ -25,7 +25,9 @@ export const db = {
|
|||
const tableNames =
|
||||
entities.length > 0
|
||||
? entities.map((entity) => em.getRepository(entity).metadata.tableName)
|
||||
: dataSource.entityMetadatas.map((entity) => entity.tableName);
|
||||
: dataSource.entityMetadatas
|
||||
.map((entity) => entity.tableName)
|
||||
.filter((tableName) => !tableName.startsWith('geodata'));
|
||||
|
||||
let deleteUsers = false;
|
||||
for (const tableName of tableNames) {
|
||||
|
|
|
|||
24
web/src/api/open-api/api.ts
generated
24
web/src/api/open-api/api.ts
generated
|
|
@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto {
|
|||
*/
|
||||
'existingIds': Array<string>;
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @export
|
||||
* @enum {string}
|
||||
*/
|
||||
|
||||
export const CitiesFile = {
|
||||
Cities15000: 'cities15000',
|
||||
Cities5000: 'cities5000',
|
||||
Cities1000: 'cities1000',
|
||||
Cities500: 'cities500'
|
||||
} as const;
|
||||
|
||||
export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile];
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @export
|
||||
|
|
@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto {
|
|||
* @interface SystemConfigReverseGeocodingDto
|
||||
*/
|
||||
export interface SystemConfigReverseGeocodingDto {
|
||||
/**
|
||||
*
|
||||
* @type {CitiesFile}
|
||||
* @memberof SystemConfigReverseGeocodingDto
|
||||
*/
|
||||
'citiesFileOverride': CitiesFile;
|
||||
/**
|
||||
*
|
||||
* @type {boolean}
|
||||
|
|
@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto {
|
|||
*/
|
||||
'enabled': boolean;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* @export
|
||||
|
|
|
|||
|
|
@ -4,13 +4,12 @@
|
|||
NotificationType,
|
||||
} from '$lib/components/shared-components/notification/notification';
|
||||
import { handleError } from '$lib/utils/handle-error';
|
||||
import { api, CitiesFile, SystemConfigDto } from '@api';
|
||||
import { api, SystemConfigDto } from '@api';
|
||||
import { cloneDeep, isEqual } from 'lodash-es';
|
||||
import { fade } from 'svelte/transition';
|
||||
import SettingAccordion from '../setting-accordion.svelte';
|
||||
import SettingButtonsRow from '../setting-buttons-row.svelte';
|
||||
import SettingSwitch from '../setting-switch.svelte';
|
||||
import SettingSelect from '../setting-select.svelte';
|
||||
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
|
||||
|
||||
export let config: SystemConfigDto; // this is the config that is being edited
|
||||
|
|
@ -39,7 +38,6 @@
|
|||
},
|
||||
reverseGeocoding: {
|
||||
enabled: config.reverseGeocoding.enabled,
|
||||
citiesFileOverride: config.reverseGeocoding.citiesFileOverride,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
|
@ -131,24 +129,6 @@
|
|||
subtitle="Enable reverse geocoding"
|
||||
bind:checked={config.reverseGeocoding.enabled}
|
||||
/>
|
||||
|
||||
<hr />
|
||||
|
||||
<SettingSelect
|
||||
label="Precision"
|
||||
desc="Set reverse geocoding precision"
|
||||
name="reverse-geocoding-precision"
|
||||
bind:value={config.reverseGeocoding.citiesFileOverride}
|
||||
options={[
|
||||
{ value: CitiesFile.Cities500, text: 'Cities with more than 500 people' },
|
||||
{ value: CitiesFile.Cities1000, text: 'Cities with more than 1000 people' },
|
||||
{ value: CitiesFile.Cities5000, text: 'Cities with more than 5000 people' },
|
||||
{ value: CitiesFile.Cities15000, text: 'Cities with more than 15000 people' },
|
||||
]}
|
||||
disabled={disabled || !config.reverseGeocoding.enabled}
|
||||
isEdited={config.reverseGeocoding.citiesFileOverride !==
|
||||
savedConfig.reverseGeocoding.citiesFileOverride}
|
||||
/>
|
||||
</div></SettingAccordion
|
||||
>
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue