feat: play around with e2e tests

.github/workflows/test.yml

@@ -21,9 +21,19 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "21.0.0-nightly20230921480ab8c3a4"
+
       - name: Run npm install
         run: npm ci
 
+      - name: Checkout test assets
+        uses: actions/checkout@v4
+        with:
+          repository: etnoy/immich-test-assets
+          path: ./test/assets
+
       - name: Run e2e tests
         run: npm run test:e2e
         if: ${{ !cancelled() }}

server/package-lock.json

@@ -60,6 +60,7 @@
       },
       "devDependencies": {
         "@nestjs/cli": "^10.1.16",
+        "@nestjs/microservices": "^10.2.5",
         "@nestjs/schematics": "^10.0.2",
         "@nestjs/testing": "^10.2.2",
         "@openapitools/openapi-generator-cli": "2.7.0",
@@ -2117,6 +2118,64 @@
         }
       }
     },
+    "node_modules/@nestjs/microservices": {
+      "version": "10.2.5",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-10.2.5.tgz",
+      "integrity": "sha512-oeBR2Tpg9zT0VL84nyH+bjsXlpDlKreWnYPwASgW1Oe/LqaeGhBKpbmYcV9qrD+QKGREHbz1zAffktxpNcnJ9w==",
+      "devOptional": true,
+      "dependencies": {
+        "iterare": "1.2.1",
+        "tslib": "2.6.2"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nest"
+      },
+      "peerDependencies": {
+        "@grpc/grpc-js": "*",
+        "@nestjs/common": "^10.0.0",
+        "@nestjs/core": "^10.0.0",
+        "@nestjs/websockets": "^10.0.0",
+        "amqp-connection-manager": "*",
+        "amqplib": "*",
+        "cache-manager": "*",
+        "ioredis": "*",
+        "kafkajs": "*",
+        "mqtt": "*",
+        "nats": "*",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^7.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@grpc/grpc-js": {
+          "optional": true
+        },
+        "@nestjs/websockets": {
+          "optional": true
+        },
+        "amqp-connection-manager": {
+          "optional": true
+        },
+        "amqplib": {
+          "optional": true
+        },
+        "cache-manager": {
+          "optional": true
+        },
+        "ioredis": {
+          "optional": true
+        },
+        "kafkajs": {
+          "optional": true
+        },
+        "mqtt": {
+          "optional": true
+        },
+        "nats": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@nestjs/platform-express": {
       "version": "10.2.2",
       "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.2.2.tgz",
@@ -2520,6 +2579,74 @@
       "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==",
       "dev": true
     },
+    "node_modules/@openapitools/openapi-generator-cli/node_modules/@nestjs/microservices": {
+      "version": "9.4.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-9.4.3.tgz",
+      "integrity": "sha512-piMw8d3C4ppc5St5AhQEtecMhyeBK2Q1VYk4AL3NKtG6U0fzz/6KLiETpWdKXmazeI/m7qac2upOvwmRzle0aA==",
+      "dev": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "iterare": "1.2.1",
+        "tslib": "2.5.3"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nest"
+      },
+      "peerDependencies": {
+        "@grpc/grpc-js": "*",
+        "@nestjs/common": "^9.0.0",
+        "@nestjs/core": "^9.0.0",
+        "@nestjs/websockets": "^9.0.0",
+        "amqp-connection-manager": "*",
+        "amqplib": "*",
+        "cache-manager": "*",
+        "ioredis": "*",
+        "kafkajs": "*",
+        "mqtt": "*",
+        "nats": "*",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^7.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@grpc/grpc-js": {
+          "optional": true
+        },
+        "@nestjs/websockets": {
+          "optional": true
+        },
+        "amqp-connection-manager": {
+          "optional": true
+        },
+        "amqplib": {
+          "optional": true
+        },
+        "cache-manager": {
+          "optional": true
+        },
+        "ioredis": {
+          "optional": true
+        },
+        "kafkajs": {
+          "optional": true
+        },
+        "mqtt": {
+          "optional": true
+        },
+        "nats": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@openapitools/openapi-generator-cli/node_modules/@nestjs/microservices/node_modules/tslib": {
+      "version": "2.5.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
+      "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==",
+      "dev": true,
+      "optional": true,
+      "peer": true
+    },
     "node_modules/@openapitools/openapi-generator-cli/node_modules/@nestjs/platform-express": {
       "version": "9.4.3",
       "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",
@@ -15674,6 +15801,16 @@
       "integrity": "sha512-V0izw6tWs6fTp9+KiiPUbGHWALy563Frn8X6Bm87ANLRuE46iuBMD5acKBDP5lKL/75QFvrzSJT7HkCbB0jTpg==",
       "requires": {}
     },
+    "@nestjs/microservices": {
+      "version": "10.2.5",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-10.2.5.tgz",
+      "integrity": "sha512-oeBR2Tpg9zT0VL84nyH+bjsXlpDlKreWnYPwASgW1Oe/LqaeGhBKpbmYcV9qrD+QKGREHbz1zAffktxpNcnJ9w==",
+      "devOptional": true,
+      "requires": {
+        "iterare": "1.2.1",
+        "tslib": "2.6.2"
+      }
+    },
     "@nestjs/platform-express": {
       "version": "10.2.2",
       "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.2.2.tgz",
@@ -15903,6 +16040,28 @@
             }
           }
         },
+        "@nestjs/microservices": {
+          "version": "9.4.3",
+          "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-9.4.3.tgz",
+          "integrity": "sha512-piMw8d3C4ppc5St5AhQEtecMhyeBK2Q1VYk4AL3NKtG6U0fzz/6KLiETpWdKXmazeI/m7qac2upOvwmRzle0aA==",
+          "dev": true,
+          "optional": true,
+          "peer": true,
+          "requires": {
+            "iterare": "1.2.1",
+            "tslib": "2.5.3"
+          },
+          "dependencies": {
+            "tslib": {
+              "version": "2.5.3",
+              "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
+              "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==",
+              "dev": true,
+              "optional": true,
+              "peer": true
+            }
+          }
+        },
         "@nestjs/platform-express": {
           "version": "9.4.3",
           "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",

server/package.json

@@ -61,8 +61,8 @@
     "exiftool-vendored": "^23.0.0",
     "exiftool-vendored.pl": "^12.62.0",
     "fluent-ffmpeg": "^2.1.2",
-    "glob": "^10.3.3",
     "geo-tz": "^7.0.7",
+    "glob": "^10.3.3",
     "handlebars": "^4.7.8",
     "i18n-iso-countries": "^7.6.0",
     "immich": "^0.41.0",
@@ -86,6 +86,7 @@
   },
   "devDependencies": {
     "@nestjs/cli": "^10.1.16",
+    "@nestjs/microservices": "^10.2.5",
     "@nestjs/schematics": "^10.0.2",
     "@nestjs/testing": "^10.2.2",
     "@openapitools/openapi-generator-cli": "2.7.0",

server/src/domain/job/job.repository.ts

@@ -111,4 +111,5 @@ export interface IJobRepository {
   empty(name: QueueName): Promise<void>;
   getQueueStatus(name: QueueName): Promise<QueueStatus>;
   getJobCounts(name: QueueName): Promise<JobCounts>;
+  obliterate(name: QueueName, force: boolean): Promise<void>;
 }

server/src/domain/job/job.service.ts

@@ -63,6 +63,12 @@ export class JobService {
     return response;
   }
 
+  async obliterateAll(force = false): Promise<void> {
+    for (const queueName of Object.values(QueueName)) {
+      await this.jobRepository.obliterate(queueName, force);
+    }
+  }
+
   private async start(name: QueueName, { force }: JobCommandDto): Promise<void> {
     const { isActive } = await this.jobRepository.getQueueStatus(name);
     if (isActive) {

server/src/domain/smart-info/smart-info.service.ts

@@ -43,6 +43,7 @@ export class SmartInfoService {
 
   async handleClassifyImage({ id }: IEntityJob) {
     const { machineLearning } = await this.configCore.getConfig();
+    console.log(machineLearning);
     if (!machineLearning.enabled || !machineLearning.classification.enabled) {
       return true;
     }

server/src/infra/repositories/job.repository.ts

@@ -49,6 +49,10 @@ export class JobRepository implements IJobRepository {
     return this.getQueue(name).drain();
   }
 
+  obliterate(name: QueueName, force = false) {
+    return this.getQueue(name).obliterate({force});
+  }
+
   getJobCounts(name: QueueName): Promise<JobCounts> {
     return this.getQueue(name).getJobCounts(
       'active',

server/src/microservices/processors/metadata-extraction.processor.ts

@@ -76,9 +76,10 @@ export class MetadataExtractionProcessor {
         await this.geocodingRepository.deleteCache();
       }
       this.logger.log('Initializing Reverse Geocoding');
-
       await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
-      await this.geocodingRepository.init();
+
+      //await this.geocodingRepository.init();
+
       await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
 
       this.logger.log('Reverse Geocoding Initialized');

server/test/api/index.ts

@@ -1,6 +1,7 @@
 import { albumApi } from './album-api';
 import { assetApi } from './asset-api';
 import { authApi } from './auth-api';
+import { jobApi } from './job-api';
 import { libraryApi } from './library-api';
 import { sharedLinkApi } from './shared-link-api';
 import { userApi } from './user-api';
@@ -12,4 +13,5 @@ export const api = {
   sharedLinkApi,
   albumApi,
   userApi,
+  jobApi,
 };

server/test/api/job-api.ts

@@ -0,0 +1,11 @@
+import { AllJobStatusResponseDto, CreateLibraryDto, LibraryResponseDto, ScanLibraryDto } from '@app/domain';
+import { send } from 'process';
+import request from 'supertest';
+
+export const jobApi = {
+  getAllJobsStatus: async (server: any, accessToken: string) => {
+    const { body, status } = await request(server).get(`/jobs/`).set('Authorization', `Bearer ${accessToken}`);
+    expect(status).toBe(200);
+    return body as AllJobStatusResponseDto;
+  },
+};

server/test/api/library-api.ts

@@ -1,4 +1,5 @@
-import { LibraryResponseDto } from '@app/domain';
+import { CreateLibraryDto, LibraryResponseDto, LibraryStatsResponseDto, ScanLibraryDto } from '@app/domain';
+import { send } from 'process';
 import request from 'supertest';
 
 export const libraryApi = {
@@ -7,4 +8,29 @@ export const libraryApi = {
     expect(status).toBe(200);
     return body as LibraryResponseDto[];
   },
+
+  createLibrary: async (server: any, accessToken: string, dto: CreateLibraryDto) => {
+    const { body, status } = await request(server)
+      .post(`/library/`)
+      .set('Authorization', `Bearer ${accessToken}`)
+      .send(dto);
+    expect(status).toBe(201);
+    return body as LibraryResponseDto;
+  },
+
+  scanLibrary: async (server: any, accessToken: string, id: string, dto: ScanLibraryDto) => {
+    const { body, status } = await request(server)
+      .post(`/library/${id}/scan`)
+      .set('Authorization', `Bearer ${accessToken}`)
+      .send(dto);
+    expect(status).toBe(201);
+  },
+
+  getLibraryStatistics: async (server: any, accessToken: string, id: string): Promise<LibraryStatsResponseDto> => {
+    const { body, status } = await request(server)
+      .get(`/library/${id}/statistics`)
+      .set('Authorization', `Bearer ${accessToken}`);
+    expect(status).toBe(200);
+    return body;
+  },
 };

server/test/e2e/library2.e2e-spec.ts

@@ -0,0 +1,170 @@
+import {
+  AuthService,
+  AuthUserDto,
+  ISystemConfigRepository,
+  JobCommand,
+  JobService,
+  LibraryService,
+  QueueName,
+  SystemConfigCore,
+} from '@app/domain';
+import { AssetService } from '@app/immich/api-v1/asset/asset.service';
+import { AppModule } from '@app/immich/app.module';
+import { AppService } from '@app/immich/app.service';
+import { RedisIoAdapter } from '@app/infra';
+import { LibraryType } from '@app/infra/entities';
+import { INestApplication, Logger } from '@nestjs/common';
+import { ClientProxy, ClientsModule, Transport } from '@nestjs/microservices';
+import { Test, TestingModule } from '@nestjs/testing';
+import { api } from '@test/api';
+import { db } from '@test/db';
+import { sleep } from '@test/test-utils';
+import { AppService as MicroAppService } from 'src/microservices/app.service';
+import { bootstrap } from 'src/microservices/main';
+
+import { MicroservicesModule } from 'src/microservices/microservices.module';
+
+describe('libe2e', () => {
+  let app: INestApplication;
+
+  let authService: AuthService;
+  let appService: AppService;
+  let assetService: AssetService;
+
+  let microServices: INestApplication;
+
+  let libraryService: LibraryService;
+  let jobService: JobService;
+  let microAppService: MicroAppService;
+
+  let adminUser: AuthUserDto;
+
+  let server: any;
+
+  let moduleFixture: TestingModule;
+  let microFixture: TestingModule;
+
+  beforeAll(async () => {
+    jest.useRealTimers();
+
+    moduleFixture = await Test.createTestingModule({
+      imports: [
+        AppModule,
+        ClientsModule.register([
+          {
+            name: 'microservices',
+            transport: Transport.REDIS,
+            options: {
+              host: process.env.REDIS_HOSTNAME,
+              port: Number(process.env.REDIS_PORT),
+            },
+          },
+        ]),
+      ],
+    })
+      //.setLogger(new Logger())
+      .compile();
+
+    microFixture = await Test.createTestingModule({
+      imports: [
+        MicroservicesModule,
+        ClientsModule.register([
+          {
+            name: 'microservices',
+            transport: Transport.REDIS,
+            options: {
+              host: process.env.REDIS_HOSTNAME,
+              port: Number(process.env.REDIS_PORT),
+            },
+          },
+        ]),
+      ],
+    })
+      //  .setLogger(new Logger())
+      .compile();
+
+    const configCore = new SystemConfigCore(moduleFixture.get(ISystemConfigRepository));
+    let config = await configCore.getConfig();
+    config.machineLearning.enabled = false;
+    console.log(config);
+    await configCore.updateConfig(config);
+
+    microServices = microFixture.createNestApplication();
+
+    await microServices.init();
+
+    app = moduleFixture.createNestApplication();
+    server = app.getHttpServer();
+
+    await app.init();
+
+    await app.startAllMicroservices();
+
+    await app.init();
+
+    jobService = moduleFixture.get(JobService);
+
+    await microFixture.get(MicroAppService).init();
+  });
+
+  describe('can import library', () => {
+    beforeAll(async () => {
+      await db.reset();
+      await jobService.obliterateAll(true);
+
+      await api.authApi.adminSignUp(server);
+      const admin = await api.authApi.adminLogin(server);
+      await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' });
+
+      const library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+        type: LibraryType.EXTERNAL,
+        name: 'Library',
+        importPaths: [`${__dirname}/../assets/nature`],
+        exclusionPatterns: [],
+      });
+
+      // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      let isFinished = false;
+      // TODO: this shouldn't be a while loop
+      while (!isFinished) {
+        const jobStatus = await api.jobApi.getAllJobsStatus(server, admin.accessToken);
+        console.log(jobStatus);
+
+        let jobsActive = false;
+        Object.values(jobStatus).forEach((job) => {
+          if (job.queueStatus.isActive) {
+            jobsActive = true;
+          }
+        });
+
+        if (!jobsActive && jobStatus[QueueName.LIBRARY].jobCounts.completed > 0) {
+          isFinished = true;
+        }
+        isFinished = true;
+
+        await sleep(5000);
+      }
+
+      // Library has been refreshed now
+    });
+
+    it('scans the library', async () => {
+      const assets = await assetService.getAllAssets(adminUser, {});
+      console.log(assets);
+      const jobStatus = await jobService.getAllJobsStatus();
+      console.log(jobStatus);
+
+      // Should have imported the 7 test assets
+      expect(assets).toHaveLength(7);
+    });
+  });
+
+  afterAll(async () => {
+    // await clearDb(database);
+    await app.close();
+    await microServices.close();
+  });
+});

server/test/e2e/setup.ts

@@ -18,4 +18,6 @@ export default async () => {
 
   process.env.REDIS_PORT = String(redis.getMappedPort(6379));
   process.env.REDIS_HOSTNAME = redis.getHost();
+
+  process.env.TYPESENSE_ENABLED = 'false';
 };

server/test/test-utils.ts

@@ -49,6 +49,10 @@ export function getAuthUser(): AuthUserDto {
   };
 }
 
+export function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 export const api = {
   adminSignUp: async (server: any) => {
     const { status, body } = await request(server).post('/auth/admin-sign-up').send(adminSignupStub);