fix(server): reverse geocoding crash loop (#2489)

server/apps/immich/src/main.ts

@@ -1,23 +1,23 @@
+import {
+  getLogLevels,
+  IMMICH_ACCESS_COOKIE,
+  IMMICH_API_KEY_HEADER,
+  IMMICH_API_KEY_NAME,
+  MACHINE_LEARNING_ENABLED,
+  SearchService,
+  SERVER_VERSION,
+} from '@app/domain';
+import { RedisIoAdapter } from '@app/infra';
 import { Logger } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
 import { NestExpressApplication } from '@nestjs/platform-express';
 import { DocumentBuilder, SwaggerDocumentOptions, SwaggerModule } from '@nestjs/swagger';
+import { json } from 'body-parser';
 import cookieParser from 'cookie-parser';
 import { writeFileSync } from 'fs';
 import path from 'path';
 import { AppModule } from './app.module';
-import { RedisIoAdapter } from '@app/infra';
-import { json } from 'body-parser';
 import { patchOpenAPI } from './utils/patch-open-api.util';
-import {
-  getLogLevels,
-  MACHINE_LEARNING_ENABLED,
-  SERVER_VERSION,
-  IMMICH_ACCESS_COOKIE,
-  SearchService,
-  IMMICH_API_KEY_HEADER,
-  IMMICH_API_KEY_NAME,
-} from '@app/domain';
 
 const logger = new Logger('ImmichServer');
 

server/apps/microservices/src/main.ts

@@ -4,6 +4,7 @@ import { SERVER_VERSION } from '@app/domain';
 import { getLogLevels } from '@app/domain';
 import { RedisIoAdapter } from '@app/infra';
 import { MicroservicesModule } from './microservices.module';
+import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
 
 const logger = new Logger('ImmichMicroservice');
 
@@ -16,6 +17,20 @@ async function bootstrap() {
 
   app.useWebSocketAdapter(new RedisIoAdapter(app));
 
+  const metadataService = app.get(MetadataExtractionProcessor);
+
+  process.on('uncaughtException', (error: Error | any) => {
+    const isCsvError = error.code === 'CSV_RECORD_INCONSISTENT_FIELDS_LENGTH';
+    if (!isCsvError) {
+      throw error;
+    }
+
+    logger.warn('Geocoding csv parse error, trying again without cache...');
+    metadataService.init(true);
+  });
+
+  await metadataService.init();
+
   await app.listen(listeningPort, () => {
     const envName = (process.env.NODE_ENV || 'development').toUpperCase();
     logger.log(
@@ -23,4 +38,5 @@ async function bootstrap() {
     );
   });
 }
+
 bootstrap();

server/apps/microservices/src/processors/metadata-extraction.processor.ts

@@ -46,16 +46,18 @@ export class MetadataExtractionProcessor {
   ) {
     this.assetCore = new AssetCore(assetRepository, jobRepository);
     this.reverseGeocodingEnabled = !configService.get('DISABLE_REVERSE_GEOCODING');
-    this.init();
   }
 
-  private async init() {
+  async init(skipCache = false) {
     this.logger.warn(`Reverse geocoding is ${this.reverseGeocodingEnabled ? 'enabled' : 'disabled'}`);
     if (!this.reverseGeocodingEnabled) {
       return;
     }
 
     try {
+      if (!skipCache) {
+        await this.geocodingRepository.deleteCache();
+      }
       this.logger.log('Initializing Reverse Geocoding');
 
       await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);

server/libs/domain/src/metadata/geocoding.repository.ts

@@ -14,4 +14,5 @@ export interface ReverseGeocodeResult {
 export interface IGeocodingRepository {
   init(): Promise<void>;
   reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
+  deleteCache(): Promise<void>;
 }

server/libs/infra/src/repositories/geocoding.repository.ts

@@ -1,8 +1,9 @@
-import { GeoPoint, ReverseGeocodeResult } from '@app/domain';
+import { GeoPoint, IGeocodingRepository, ReverseGeocodeResult } from '@app/domain';
 import { localGeocodingConfig } from '@app/infra';
 import { Injectable, Logger } from '@nestjs/common';
+import { rm } from 'fs/promises';
 import { getName } from 'i18n-iso-countries';
-import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder';
+import geocoder, { AddressObject } from 'local-reverse-geocoder';
 import { promisify } from 'util';
 
 export interface AdminCode {
@@ -16,15 +17,22 @@ export type GeoData = AddressObject & {
   admin2Code?: AdminCode | string;
 };
 
-const init = (options: InitOptions): Promise<void> => new Promise<void>((resolve) => geocoder.init(options, resolve));
+const init = (): Promise<void> => new Promise<void>((resolve) => geocoder.init(localGeocodingConfig, resolve));
 const lookup = promisify<GeoPoint[], number, AddressObject[][]>(geocoder.lookUp).bind(geocoder);
 
 @Injectable()
-export class GeocodingRepository {
+export class GeocodingRepository implements IGeocodingRepository {
   private logger = new Logger(GeocodingRepository.name);
 
   async init(): Promise<void> {
-    await init(localGeocodingConfig);
+    await init();
+  }
+
+  async deleteCache() {
+    const dumpDirectory = localGeocodingConfig.dumpDirectory;
+    if (dumpDirectory) {
+      await rm(dumpDirectory, { recursive: true, force: true });
+    }
   }
 
   async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {