install docker dependencies

.github/workflows/test.yml

@@ -17,6 +17,10 @@ jobs:
       run:
         working-directory: ./server
 
+    env:
+      ENV LD_LIBRARY_PATH: /usr/local/lib:$LD_LIBRARY_PATH
+      ENV LD_RUN_PATH: /usr/local/lib:$LD_RUN_PATH
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -30,6 +34,12 @@ jobs:
         with:
           node-version: "21.0.0-nightly20230921480ab8c3a4"
 
+      - name: Install dependencies
+        run: sudo apt-get update && apt-get install -yqq build-essential ninja-build meson pkg-config jq zlib1g autoconf libglib2.0-dev libexpat1-dev librsvg2-dev libexif-dev libwebp-dev liborc-0.4-dev libjpeg62-turbo-dev libgsf-1-dev libspng-dev libjxl-dev libheif-dev liblcms2-2 mesa-va-drivers libmimalloc2.0 $(if [ $(arch) = "x86_64" ]; then echo "intel-media-va-driver-non-free"; fi) && ./install-ffmpeg.sh && apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+      - name: Install libraw, imagemagick, libvips
+        run: ./bin/build-libraw.sh && ./bin/build-imagemagick.sh && ./bin/build-libvips.sh
+
       - name: Checkout test assets
         uses: actions/checkout@v4
         with:
@@ -37,7 +47,7 @@ jobs:
           path: ./server/test/assets
 
       - name: Run e2e tests
-        run: NODE_OPTIONS='--experimental-vm-modules' npm run test:e2e
+        run: npm run test:e2e -- --forceExit
         if: ${{ !cancelled() }}
 
   doc-tests:

server/package.json

@@ -26,7 +26,7 @@
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "jest --config test/e2e/jest-e2e.json --runInBand --detectOpenHandles --verbose",
+    "test:e2e": "NODE_OPTIONS='--experimental-vm-modules' jest --config test/e2e/jest-e2e.json --runInBand",
     "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js",
     "typeorm:migrations:create": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:create",
     "typeorm:migrations:generate": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:generate -d ./src/infra/database.config.ts",

server/src/domain/job/job.repository.ts

@@ -112,5 +112,4 @@ export interface IJobRepository {
   getQueueStatus(name: QueueName): Promise<QueueStatus>;
   getJobCounts(name: QueueName): Promise<JobCounts>;
   obliterate(name: QueueName, force: boolean): Promise<void>;
-  closeWorkers(): Promise<void>;
 }

server/src/domain/job/job.service.ts

@@ -69,10 +69,6 @@ export class JobService {
     }
   }
 
-  async closeAll(): Promise<void> {
-    await this.jobRepository.closeWorkers();
-  }
-
   private async start(name: QueueName, { force }: JobCommandDto): Promise<void> {
     const { isActive } = await this.jobRepository.getQueueStatus(name);
     if (isActive) {

server/src/domain/library/library.service.ts

@@ -350,8 +350,6 @@ export class LibraryService {
   }
 
   async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise<boolean> {
-    console.log('Handle queue asset refresh: ' + JSON.stringify(job));
-    console.log(await this.repository.getAll(true, LibraryType.EXTERNAL));
     const library = await this.repository.get(job.id);
     if (!library || library.type !== LibraryType.EXTERNAL) {
       this.logger.warn('Can only refresh external libraries');

server/src/infra/repositories/job.repository.ts

@@ -28,16 +28,6 @@ export class JobRepository implements IJobRepository {
     worker.concurrency = concurrency;
   }
 
-  async closeWorkers() {
-    for (const queue in Object.keys(this.workers)) {
-      const queueName = queue as QueueName;
-      const worker = this.workers[queueName];
-      if (worker) {
-        await worker.close();
-      }
-    }
-  }
-
   async getQueueStatus(name: QueueName): Promise<QueueStatus> {
     const queue = this.getQueue(name);
 

server/src/microservices/app.service.ts

@@ -44,8 +44,10 @@ export class AppService {
 
   async init(clearBeforeStart = false) {
     if (clearBeforeStart) {
+      // Clear all jobs on application startup, mainly used for e2e testing
       await this.jobService.obliterateAll(true);
     }
+
     await this.jobService.registerHandlers({
       [JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
       [JobName.CLEAN_OLD_AUDIT_LOGS]: () => this.auditService.handleCleanup(),

server/src/microservices/processors/metadata-extraction.processor.ts

@@ -76,10 +76,9 @@ export class MetadataExtractionProcessor {
         await this.geocodingRepository.deleteCache();
       }
       this.logger.log('Initializing Reverse Geocoding');
+
       await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
-
-      //await this.geocodingRepository.init();
-
+      await this.geocodingRepository.init();
       await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
 
       this.logger.log('Reverse Geocoding Initialized');

server/test/e2e/formats.e2e-spec.ts

@@ -0,0 +1,93 @@
+import { JobService, LoginResponseDto, QueueName } from '@app/domain';
+import { AppModule } from '@app/immich/app.module';
+import { LibraryType } from '@app/infra/entities';
+import { INestApplication, Logger } from '@nestjs/common';
+import { Test, TestingModule } from '@nestjs/testing';
+import { api } from '@test/api';
+import { db } from '@test/db';
+import { waitForQueues } from '@test/test-utils';
+import { AppService as MicroAppService } from 'src/microservices/app.service';
+
+import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor';
+
+describe('File format (e2e)', () => {
+  let app: INestApplication;
+  let jobService: JobService;
+  let server: any;
+  let moduleFixture: TestingModule;
+  let admin: LoginResponseDto;
+
+  beforeAll(async () => {
+    jest.useRealTimers();
+
+    moduleFixture = await Test.createTestingModule({
+      imports: [AppModule],
+      providers: [MetadataExtractionProcessor, MicroAppService],
+    })
+      .setLogger(new Logger())
+      .compile();
+
+    app = moduleFixture.createNestApplication();
+
+    await app.init();
+    app.enableShutdownHooks();
+    server = app.getHttpServer();
+
+    jobService = moduleFixture.get(JobService);
+
+    await moduleFixture.get(MicroAppService).init(true);
+  });
+
+  beforeEach(async () => {
+    // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder
+
+    await db.reset();
+
+    await jobService.obliterateAll(true);
+
+    await api.authApi.adminSignUp(server);
+    admin = await api.authApi.adminLogin(server);
+    await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' });
+  });
+
+  it('should import a jpg file', async () => {
+    const library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+      type: LibraryType.EXTERNAL,
+      name: 'Library',
+      importPaths: [`${__dirname}/../assets/formats/jpg`],
+      exclusionPatterns: [],
+    });
+
+    await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+    await waitForQueues(jobService);
+
+    const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+    expect(assets).toHaveLength(1);
+  });
+
+  it('should import a heic file', async () => {
+    const library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+      type: LibraryType.EXTERNAL,
+      name: 'Library',
+      importPaths: [`${__dirname}/../assets/formats/heic`],
+      exclusionPatterns: [],
+    });
+
+    await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+    await waitForQueues(jobService);
+
+    const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+    expect(assets).toHaveLength(1);
+    console.log(assets);
+  });
+
+  afterAll(async () => {
+    console.log(await jobService.getAllJobsStatus());
+    await jobService.obliterateAll(true);
+    await app.close();
+    await moduleFixture.close();
+    await db.disconnect();
+  });
+});

server/test/e2e/library2.e2e-spec.ts

@@ -1,25 +1,20 @@
 import { JobService, LoginResponseDto, QueueName } from '@app/domain';
 import { AppModule } from '@app/immich/app.module';
 import { LibraryType } from '@app/infra/entities';
-import { JobRepository } from '@app/infra/repositories';
 import { INestApplication, Logger } from '@nestjs/common';
 import { Test, TestingModule } from '@nestjs/testing';
 import { api } from '@test/api';
 import { db } from '@test/db';
-import { sleep } from '@test/test-utils';
+import { waitForQueues } from '@test/test-utils';
 import { AppService as MicroAppService } from 'src/microservices/app.service';
 
 import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor';
 
-describe('libe2e', () => {
+describe('Library queue e2e', () => {
   let app: INestApplication;
-
   let jobService: JobService;
-
   let server: any;
-
   let moduleFixture: TestingModule;
-
   let admin: LoginResponseDto;
 
   beforeAll(async () => {
@@ -44,7 +39,9 @@ describe('libe2e', () => {
   });
 
   describe('can import library', () => {
-    beforeAll(async () => {
+    beforeEach(async () => {
+      // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder
+
       await db.reset();
 
       await jobService.obliterateAll(true);
@@ -52,7 +49,9 @@ describe('libe2e', () => {
       await api.authApi.adminSignUp(server);
       admin = await api.authApi.adminLogin(server);
       await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' });
+    });
 
+    it('should scan the whole folder', async () => {
       const library = await api.libraryApi.createLibrary(server, admin.accessToken, {
         type: LibraryType.EXTERNAL,
         name: 'Library',
@@ -60,45 +59,34 @@ describe('libe2e', () => {
         exclusionPatterns: [],
       });
 
-      console.log(await api.libraryApi.getAll(server, admin.accessToken));
-
-      // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder
-
       await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
 
-      let isFinished = false;
-      // TODO: this shouldn't be a while loop
-      while (!isFinished) {
-        const jobStatus = await api.jobApi.getAllJobsStatus(server, admin.accessToken);
+      await waitForQueues(jobService);
 
-        let jobsActive = false;
-        Object.values(jobStatus).forEach((job) => {
-          if (job.queueStatus.isActive) {
-            jobsActive = true;
-          }
-          if (job.queueStatus.active > 0 || job.queueStatus.waiting > 0) {
-            jobsActive = true;
-          }
-        });
-
-        if (!jobsActive) {
-          isFinished = true;
-        }
-
-        await sleep(200);
-      }
-    });
-
-    it('scans the library', async () => {
       const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
       expect(assets).toHaveLength(7);
     });
+
+    it('scan with exclusions', async () => {
+      const library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+        type: LibraryType.EXTERNAL,
+        name: 'Library',
+        importPaths: [`${__dirname}/../assets/nature/`],
+        exclusionPatterns: ['**/*o*/**', '**/*c*/**'],
+      });
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      await waitForQueues(jobService);
+
+      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+
+      expect(assets).toHaveLength(1);
+      expect(assets[0].originalFileName).toBe('silver_fir');
+    });
   });
 
-  afterEach(async () => {});
-
   afterAll(async () => {
-    await jobService.closeAll();
     await jobService.obliterateAll(true);
     await app.close();
     await moduleFixture.close();

server/test/repositories/job.repository.mock.ts

@@ -11,6 +11,5 @@ export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
     getQueueStatus: jest.fn(),
     getJobCounts: jest.fn(),
     obliterate: jest.fn(),
-    closeWorkers: jest.fn(),
   };
 };

server/test/test-utils.ts

@@ -4,6 +4,7 @@ import {
   AuthDeviceResponseDto,
   AuthUserDto,
   CreateUserDto,
+  JobService,
   LibraryResponseDto,
   LoginCredentialDto,
   LoginResponseDto,
@@ -53,6 +54,30 @@ export function sleep(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+export async function waitForQueues(jobService: JobService) {
+  let isFinished = false;
+  // TODO: this shouldn't be a while loop
+  while (!isFinished) {
+    const jobStatus = await jobService.getAllJobsStatus();
+
+    let jobsActive = false;
+    Object.values(jobStatus).forEach((job) => {
+      if (job.queueStatus.isActive) {
+        jobsActive = true;
+      }
+      if (job.queueStatus.active > 0 || job.queueStatus.waiting > 0) {
+        jobsActive = true;
+      }
+    });
+
+    if (!jobsActive) {
+      isFinished = true;
+    }
+
+    await sleep(100);
+  }
+}
+
 export const api = {
   adminSignUp: async (server: any) => {
     const { status, body } = await request(server).post('/auth/admin-sign-up').send(adminSignupStub);