fix

.github/workflows/test.yml

@@ -12,21 +12,45 @@ concurrency:
 jobs:
   e2e-tests:
     name: Run end-to-end test suites
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         working-directory: ./server
 
+    env:
+      ENV LD_LIBRARY_PATH: /usr/local/lib:$LD_LIBRARY_PATH
+      ENV LD_RUN_PATH: /usr/local/lib:$LD_RUN_PATH
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - uses: actions/setup-node@v3
+
       - name: Run npm install
         run: npm ci
 
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "21.0.0-nightly20230921480ab8c3a4"
+
+      - name: Install dependencies
+        run: sudo apt-get update && sudo apt-get install -yqq build-essential ninja-build meson pkg-config jq zlib1g autoconf libglib2.0-dev libexpat1-dev librsvg2-dev libexif-dev libwebp-dev liborc-0.4-dev libjpeg-turbo8-dev libgsf-1-dev libheif-dev liblcms2-2 mesa-va-drivers libmimalloc2.0 $(if [ $(arch) = "x86_64" ]; then echo "intel-media-va-driver-non-free"; fi) && sudo FFMPEG_PLATFORM=jammy ./bin/install-ffmpeg.sh && sudo apt-get autoremove && sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/*
+
+      - name: Copy build files
+        run: mkdir build && cp bin/build-libraw.sh bin/build-imagemagick.sh bin/build-libvips.sh bin/use-camera-wb.patch build-lock.json build
+
+      - name: Install libraw, imagemagick, libvips
+        run: cd build && sudo sudo ./build-libraw.sh && sudo ./build-imagemagick.sh && sudo ./build-libvips.sh
+
+      - name: Checkout test assets
+        uses: actions/checkout@v4
+        with:
+          repository: etnoy/immich-test-assets
+          path: ./server/test/assets
+
       - name: Run e2e tests
-        run: npm run test:e2e
-        if: ${{ !cancelled() }}
+        run: npm run test:e2e -- --forceExit --detectOpenHandles
 
   doc-tests:
     name: Run documentation checks

docker/docker-compose.e2e.yml

@@ -0,0 +1,146 @@
+version: "3.8"
+
+services:
+  immich-server:
+    container_name: immich_server-e2e
+    image: immich-server-dev:latest
+    build:
+      context: ../server
+      dockerfile: Dockerfile
+      target: builder
+    command: npm run start:debug immich
+    volumes:
+      - ../server:/usr/src/app
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+      - /usr/src/app/node_modules
+    ports:
+      - 3001:3001
+      - 9230:9230
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=development
+    depends_on:
+      - redis
+      - database
+      - typesense
+
+  immich-machine-learning:
+    container_name: immich_machine_learning-e2e
+    image: immich-machine-learning-dev:latest
+    build:
+      context: ../machine-learning
+      dockerfile: Dockerfile
+    ports:
+      - 3003:3003
+    volumes:
+      - ../machine-learning:/usr/src/app
+      - model-cache:/cache
+    env_file:
+      - .env
+    environment:
+      - NODE_ENV=development
+    depends_on:
+      - database
+    restart: unless-stopped
+
+  immich-microservices:
+    container_name: immich_microservices-e2e
+    image: immich-microservices:latest
+    # extends:
+    #   file: hwaccel.yml
+    #   service: hwaccel
+    build:
+      context: ../server
+      dockerfile: Dockerfile
+      target: builder
+    command: npm run start:debug microservices
+    volumes:
+      - ../server:/usr/src/app
+      - ${UPLOAD_LOCATION}:/usr/src/app/upload
+      - /usr/src/app/node_modules
+    env_file:
+      - .env
+    ports:
+      - 9231:9230
+    environment:
+      - NODE_ENV=development
+    depends_on:
+      - database
+      - immich-server
+      - typesense
+
+  immich-web:
+    container_name: immich_web-e2e
+    image: immich-web-dev:1.9.0
+    build:
+      context: ../web
+      dockerfile: Dockerfile
+      target: dev
+    command: npm run dev --host
+    env_file:
+      - .env
+    environment:
+      # Rename these values for svelte public interface
+      - PUBLIC_IMMICH_SERVER_URL=${IMMICH_SERVER_URL}
+      - PUBLIC_IMMICH_API_URL_EXTERNAL=${IMMICH_API_URL_EXTERNAL}
+    ports:
+      - 3000:3000
+      - 24678:24678
+    volumes:
+      - ../web:/usr/src/app
+      - /usr/src/app/node_modules
+    restart: unless-stopped
+    depends_on:
+      - immich-server
+
+  typesense:
+    container_name: immich_typesense-e2e
+    image: typesense/typesense:0.24.1@sha256:9bcff2b829f12074426ca044b56160ca9d777a0c488303469143dd9f8259d4dd
+    environment:
+      - TYPESENSE_API_KEY=${TYPESENSE_API_KEY}
+      - TYPESENSE_DATA_DIR=/data
+      # remove this to get debug messages
+      - GLOG_minloglevel=1
+    volumes:
+      - tsdata:/data
+
+  redis:
+    container_name: immich_redis-e2e
+    image: redis:6.2-alpine@sha256:70a7a5b641117670beae0d80658430853896b5ef269ccf00d1827427e3263fa3
+
+  database:
+    container_name: immich_postgres-e2e
+    image: postgres:14-alpine@sha256:28407a9961e76f2d285dc6991e8e48893503cc3836a4755bbc2d40bcc272a441
+    env_file:
+      - .env
+    environment:
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_USER: ${DB_USERNAME}
+      POSTGRES_DB: ${DB_DATABASE_NAME}
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+
+  immich-proxy:
+    container_name: immich_proxy-e2e
+    image: immich-proxy-dev:latest
+    environment:
+      # Make sure these values get passed through from the env file
+      - IMMICH_SERVER_URL
+      - IMMICH_WEB_URL
+    build:
+      context: ../nginx
+      dockerfile: Dockerfile
+    ports:
+      - 2285:8080
+    depends_on:
+      - immich-server
+      - immich-web
+    restart: unless-stopped
+
+volumes:
+  pgdata:
+  model-cache:
+  tsdata:

server/bin/install-ffmpeg.sh

@@ -2,17 +2,22 @@
 
 set -e
 
+: "${FFMPEG_PLATFORM:=bookworm}"
+
+echo Using platform $FFMPEG_PLATFORM
+
 LOCK=$(jq -c '.packages[] | select(.name == "ffmpeg")' build-lock.json)
-export TARGETARCH=${TARGETARCH:=$(dpkg --print-architecture)}
+TARGETARCH=${TARGETARCH:=$(dpkg --print-architecture)}
+export PLATFORMARCH=${FFMPEG_PLATFORM}-${TARGETARCH}
 FFMPEG_VERSION=${FFMPEG_VERSION:=$(echo $LOCK | jq -r '.version')}
-FFMPEG_SHA256=${FFMPEG_SHA256:=$(echo $LOCK | jq -r '.sha256[$ENV.TARGETARCH]')}
+FFMPEG_SHA256=${FFMPEG_SHA256:=$(echo $LOCK | jq -r '.sha256[$ENV.PLATFORMARCH]')}
 
-echo "$FFMPEG_SHA256  jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb" > ffmpeg.sha256
-
-wget -nv https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v${FFMPEG_VERSION}/jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb
+echo "$FFMPEG_SHA256  jellyfin-ffmpeg6_${FFMPEG_VERSION}-${FFMPEG_PLATFORM}_${TARGETARCH}.deb" > ffmpeg.sha256
+cat ffmpeg.sha256
+wget -nv https://github.com/jellyfin/jellyfin-ffmpeg/releases/download/v${FFMPEG_VERSION}/jellyfin-ffmpeg6_${FFMPEG_VERSION}-${FFMPEG_PLATFORM}_${TARGETARCH}.deb
 sha256sum -c ffmpeg.sha256
-apt-get -yqq -f install ./jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb
-rm jellyfin-ffmpeg6_${FFMPEG_VERSION}-bookworm_${TARGETARCH}.deb
+apt-get -y -f install ./jellyfin-ffmpeg6_${FFMPEG_VERSION}-${FFMPEG_PLATFORM}_${TARGETARCH}.deb
+rm jellyfin-ffmpeg6_${FFMPEG_VERSION}-${FFMPEG_PLATFORM}_${TARGETARCH}.deb
 rm ffmpeg.sha256
 ldconfig /usr/lib/jellyfin-ffmpeg/lib
 

server/build-lock.json

@@ -19,9 +19,10 @@
       "name": "ffmpeg",
       "version": "6.0-4",
       "sha256": {
-        "amd64": "18d98b292b891cde86c2a08e5e989c3430e51a136cdc232bc4162fef3b4f0f44",
-        "arm64": "67eb1e5a38ac695dd253d9ac290ad0e9fb709e8260449a7445e8460b7db3c516",
-        "armhf": "a29605ab0eced3511c8a6623504fab5b8bb174a486d87f94bf5522ed9a5970e6"
+        "bookworm-amd64": "18d98b292b891cde86c2a08e5e989c3430e51a136cdc232bc4162fef3b4f0f44",
+        "bookworm-arm64": "67eb1e5a38ac695dd253d9ac290ad0e9fb709e8260449a7445e8460b7db3c516",
+        "bookworm-armhf": "a29605ab0eced3511c8a6623504fab5b8bb174a486d87f94bf5522ed9a5970e6",
+        "jammy-amd64": "f5c04cce7a7e6733049debbff5af7373fe8d71fb37725edb91c49a36025fea8c"
       }
     }
   ]

server/package-lock.json

@@ -60,6 +60,7 @@
       },
       "devDependencies": {
         "@nestjs/cli": "^10.1.16",
+        "@nestjs/microservices": "^10.2.5",
         "@nestjs/schematics": "^10.0.2",
         "@nestjs/testing": "^10.2.2",
         "@openapitools/openapi-generator-cli": "2.7.0",
@@ -2117,6 +2118,64 @@
         }
       }
     },
+    "node_modules/@nestjs/microservices": {
+      "version": "10.2.5",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-10.2.5.tgz",
+      "integrity": "sha512-oeBR2Tpg9zT0VL84nyH+bjsXlpDlKreWnYPwASgW1Oe/LqaeGhBKpbmYcV9qrD+QKGREHbz1zAffktxpNcnJ9w==",
+      "devOptional": true,
+      "dependencies": {
+        "iterare": "1.2.1",
+        "tslib": "2.6.2"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nest"
+      },
+      "peerDependencies": {
+        "@grpc/grpc-js": "*",
+        "@nestjs/common": "^10.0.0",
+        "@nestjs/core": "^10.0.0",
+        "@nestjs/websockets": "^10.0.0",
+        "amqp-connection-manager": "*",
+        "amqplib": "*",
+        "cache-manager": "*",
+        "ioredis": "*",
+        "kafkajs": "*",
+        "mqtt": "*",
+        "nats": "*",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^7.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@grpc/grpc-js": {
+          "optional": true
+        },
+        "@nestjs/websockets": {
+          "optional": true
+        },
+        "amqp-connection-manager": {
+          "optional": true
+        },
+        "amqplib": {
+          "optional": true
+        },
+        "cache-manager": {
+          "optional": true
+        },
+        "ioredis": {
+          "optional": true
+        },
+        "kafkajs": {
+          "optional": true
+        },
+        "mqtt": {
+          "optional": true
+        },
+        "nats": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@nestjs/platform-express": {
       "version": "10.2.2",
       "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.2.2.tgz",
@@ -2520,6 +2579,74 @@
       "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==",
       "dev": true
     },
+    "node_modules/@openapitools/openapi-generator-cli/node_modules/@nestjs/microservices": {
+      "version": "9.4.3",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-9.4.3.tgz",
+      "integrity": "sha512-piMw8d3C4ppc5St5AhQEtecMhyeBK2Q1VYk4AL3NKtG6U0fzz/6KLiETpWdKXmazeI/m7qac2upOvwmRzle0aA==",
+      "dev": true,
+      "optional": true,
+      "peer": true,
+      "dependencies": {
+        "iterare": "1.2.1",
+        "tslib": "2.5.3"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nest"
+      },
+      "peerDependencies": {
+        "@grpc/grpc-js": "*",
+        "@nestjs/common": "^9.0.0",
+        "@nestjs/core": "^9.0.0",
+        "@nestjs/websockets": "^9.0.0",
+        "amqp-connection-manager": "*",
+        "amqplib": "*",
+        "cache-manager": "*",
+        "ioredis": "*",
+        "kafkajs": "*",
+        "mqtt": "*",
+        "nats": "*",
+        "reflect-metadata": "^0.1.12",
+        "rxjs": "^7.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@grpc/grpc-js": {
+          "optional": true
+        },
+        "@nestjs/websockets": {
+          "optional": true
+        },
+        "amqp-connection-manager": {
+          "optional": true
+        },
+        "amqplib": {
+          "optional": true
+        },
+        "cache-manager": {
+          "optional": true
+        },
+        "ioredis": {
+          "optional": true
+        },
+        "kafkajs": {
+          "optional": true
+        },
+        "mqtt": {
+          "optional": true
+        },
+        "nats": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@openapitools/openapi-generator-cli/node_modules/@nestjs/microservices/node_modules/tslib": {
+      "version": "2.5.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
+      "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==",
+      "dev": true,
+      "optional": true,
+      "peer": true
+    },
     "node_modules/@openapitools/openapi-generator-cli/node_modules/@nestjs/platform-express": {
       "version": "9.4.3",
       "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",
@@ -15674,6 +15801,16 @@
       "integrity": "sha512-V0izw6tWs6fTp9+KiiPUbGHWALy563Frn8X6Bm87ANLRuE46iuBMD5acKBDP5lKL/75QFvrzSJT7HkCbB0jTpg==",
       "requires": {}
     },
+    "@nestjs/microservices": {
+      "version": "10.2.5",
+      "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-10.2.5.tgz",
+      "integrity": "sha512-oeBR2Tpg9zT0VL84nyH+bjsXlpDlKreWnYPwASgW1Oe/LqaeGhBKpbmYcV9qrD+QKGREHbz1zAffktxpNcnJ9w==",
+      "devOptional": true,
+      "requires": {
+        "iterare": "1.2.1",
+        "tslib": "2.6.2"
+      }
+    },
     "@nestjs/platform-express": {
       "version": "10.2.2",
       "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.2.2.tgz",
@@ -15903,6 +16040,28 @@
             }
           }
         },
+        "@nestjs/microservices": {
+          "version": "9.4.3",
+          "resolved": "https://registry.npmjs.org/@nestjs/microservices/-/microservices-9.4.3.tgz",
+          "integrity": "sha512-piMw8d3C4ppc5St5AhQEtecMhyeBK2Q1VYk4AL3NKtG6U0fzz/6KLiETpWdKXmazeI/m7qac2upOvwmRzle0aA==",
+          "dev": true,
+          "optional": true,
+          "peer": true,
+          "requires": {
+            "iterare": "1.2.1",
+            "tslib": "2.5.3"
+          },
+          "dependencies": {
+            "tslib": {
+              "version": "2.5.3",
+              "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
+              "integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w==",
+              "dev": true,
+              "optional": true,
+              "peer": true
+            }
+          }
+        },
         "@nestjs/platform-express": {
           "version": "9.4.3",
           "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",

server/package.json

@@ -26,7 +26,7 @@
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
-    "test:e2e": "jest --config test/e2e/jest-e2e.json --runInBand",
+    "test:e2e": "NODE_OPTIONS='--experimental-vm-modules' jest --config test/e2e/jest-e2e.json --runInBand",
     "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js",
     "typeorm:migrations:create": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:create",
     "typeorm:migrations:generate": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:generate -d ./src/infra/database.config.ts",
@@ -61,8 +61,8 @@
     "exiftool-vendored": "^23.0.0",
     "exiftool-vendored.pl": "^12.62.0",
     "fluent-ffmpeg": "^2.1.2",
-    "glob": "^10.3.3",
     "geo-tz": "^7.0.7",
+    "glob": "^10.3.3",
     "handlebars": "^4.7.8",
     "i18n-iso-countries": "^7.6.0",
     "immich": "^0.41.0",
@@ -86,6 +86,7 @@
   },
   "devDependencies": {
     "@nestjs/cli": "^10.1.16",
+    "@nestjs/microservices": "^10.2.5",
     "@nestjs/schematics": "^10.0.2",
     "@nestjs/testing": "^10.2.2",
     "@openapitools/openapi-generator-cli": "2.7.0",

server/src/domain/job/job.repository.ts

@@ -111,4 +111,5 @@ export interface IJobRepository {
   empty(name: QueueName): Promise<void>;
   getQueueStatus(name: QueueName): Promise<QueueStatus>;
   getJobCounts(name: QueueName): Promise<JobCounts>;
+  obliterate(name: QueueName, force: boolean): Promise<void>;
 }

server/src/domain/job/job.service.ts

@@ -63,6 +63,12 @@ export class JobService {
     return response;
   }
 
+  async obliterateAll(force = false): Promise<void> {
+    for (const queueName of Object.values(QueueName)) {
+      await this.jobRepository.obliterate(queueName, force);
+    }
+  }
+
   private async start(name: QueueName, { force }: JobCommandDto): Promise<void> {
     const { isActive } = await this.jobRepository.getQueueStatus(name);
     if (isActive) {

server/src/infra/repositories/job.repository.ts

@@ -49,6 +49,10 @@ export class JobRepository implements IJobRepository {
     return this.getQueue(name).drain();
   }
 
+  obliterate(name: QueueName, force = false) {
+    return this.getQueue(name).obliterate({ force });
+  }
+
   getJobCounts(name: QueueName): Promise<JobCounts> {
     return this.getQueue(name).getJobCounts(
       'active',

server/src/microservices/app.service.ts

@@ -42,7 +42,12 @@ export class AppService {
     private libraryService: LibraryService,
   ) {}
 
-  async init() {
+  async init(clearBeforeStart = false) {
+    if (clearBeforeStart) {
+      // Clear all jobs on application startup, mainly used for e2e testing
+      await this.jobService.obliterateAll(true);
+    }
+
     await this.jobService.registerHandlers({
       [JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
       [JobName.CLEAN_OLD_AUDIT_LOGS]: () => this.auditService.handleCleanup(),

server/test/api/asset-api.ts

@@ -14,6 +14,11 @@ export const assetApi = {
     expect(status).toBe(200);
     return body as AssetResponseDto;
   },
+  getAllAssets: async (server: any, accessToken: string) => {
+    const { body, status } = await request(server).get(`/asset/`).set('Authorization', `Bearer ${accessToken}`);
+    expect(status).toBe(200);
+    return body as AssetResponseDto[];
+  },
   upload: async (server: any, accessToken: string, id: string, dto: UploadDto = {}) => {
     const { content, isFavorite = false, isArchived = false } = dto;
     const { body, status } = await request(server)

server/test/api/index.ts

@@ -1,6 +1,7 @@
 import { albumApi } from './album-api';
 import { assetApi } from './asset-api';
 import { authApi } from './auth-api';
+import { jobApi } from './job-api';
 import { libraryApi } from './library-api';
 import { sharedLinkApi } from './shared-link-api';
 import { userApi } from './user-api';
@@ -12,4 +13,5 @@ export const api = {
   sharedLinkApi,
   albumApi,
   userApi,
+  jobApi,
 };

server/test/api/job-api.ts

@@ -0,0 +1,10 @@
+import { AllJobStatusResponseDto } from '@app/domain';
+import request from 'supertest';
+
+export const jobApi = {
+  getAllJobsStatus: async (server: any, accessToken: string) => {
+    const { body, status } = await request(server).get(`/jobs/`).set('Authorization', `Bearer ${accessToken}`);
+    expect(status).toBe(200);
+    return body as AllJobStatusResponseDto;
+  },
+};

server/test/api/library-api.ts

@@ -1,4 +1,4 @@
-import { LibraryResponseDto } from '@app/domain';
+import { CreateLibraryDto, LibraryResponseDto, LibraryStatsResponseDto, ScanLibraryDto } from '@app/domain';
 import request from 'supertest';
 
 export const libraryApi = {
@@ -7,4 +7,38 @@ export const libraryApi = {
     expect(status).toBe(200);
     return body as LibraryResponseDto[];
   },
+
+  createLibrary: async (server: any, accessToken: string, dto: CreateLibraryDto) => {
+    const { body, status } = await request(server)
+      .post(`/library/`)
+      .set('Authorization', `Bearer ${accessToken}`)
+      .send(dto);
+    expect(status).toBe(201);
+    return body as LibraryResponseDto;
+  },
+
+  setImportPaths: async (server: any, accessToken: string, id: string, importPaths: string[]) => {
+    const { body, status } = await request(server)
+      .put(`/library/${id}`)
+      .set('Authorization', `Bearer ${accessToken}`)
+      .send({ importPaths });
+    expect(status).toBe(200);
+    return body as LibraryResponseDto;
+  },
+
+  scanLibrary: async (server: any, accessToken: string, id: string, dto: ScanLibraryDto) => {
+    const { status } = await request(server)
+      .post(`/library/${id}/scan`)
+      .set('Authorization', `Bearer ${accessToken}`)
+      .send(dto);
+    expect(status).toBe(201);
+  },
+
+  getLibraryStatistics: async (server: any, accessToken: string, id: string): Promise<LibraryStatsResponseDto> => {
+    const { body, status } = await request(server)
+      .get(`/library/${id}/statistics`)
+      .set('Authorization', `Bearer ${accessToken}`);
+    expect(status).toBe(200);
+    return body;
+  },
 };

server/test/assets

@@ -0,0 +1 @@
+Subproject commit 20e0423a08f7dfc590db2702a9e5232d6b6e4734

server/test/db/index.ts

@@ -3,6 +3,7 @@ import { dataSource } from '@app/infra';
 export const db = {
   reset: async () => {
     if (!dataSource.isInitialized) {
+      console.log('Initializing test database...');
       await dataSource.initialize();
     }
 

server/test/e2e/formats.e2e-spec.ts

@@ -0,0 +1,128 @@
+import { JobService, LibraryResponseDto, LoginResponseDto } from '@app/domain';
+import { AppModule } from '@app/immich/app.module';
+import { RedisIoAdapter } from '@app/infra';
+import { AssetType, LibraryType } from '@app/infra/entities';
+import { INestApplication, Logger } from '@nestjs/common';
+import { Test, TestingModule } from '@nestjs/testing';
+import { api } from '@test/api';
+import { db } from '@test/db';
+import { waitForQueues } from '@test/test-utils';
+import { AppService as MicroAppService } from 'src/microservices/app.service';
+
+import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor';
+
+describe('File format (e2e)', () => {
+  let app: INestApplication;
+  let jobService: JobService;
+  let server: any;
+  let moduleFixture: TestingModule;
+  let admin: LoginResponseDto;
+
+  beforeEach(async () => {
+    // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder
+
+    await db.reset();
+
+    jest.useRealTimers();
+
+    moduleFixture = await Test.createTestingModule({
+      imports: [AppModule],
+      providers: [MetadataExtractionProcessor, MicroAppService],
+    })
+      // .setLogger(new Logger())
+      .compile();
+
+    app = moduleFixture.createNestApplication();
+
+    app.useWebSocketAdapter(new RedisIoAdapter(app));
+
+    await app.init();
+    app.enableShutdownHooks();
+    server = app.getHttpServer();
+
+    jobService = moduleFixture.get(JobService);
+
+    await moduleFixture.get(MicroAppService).init(true);
+    await jobService.obliterateAll(true);
+
+    await api.authApi.adminSignUp(server);
+    admin = await api.authApi.adminLogin(server);
+    await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' });
+  });
+
+  afterEach(async () => {
+    await jobService.obliterateAll(true);
+    await app.close();
+    await moduleFixture.close();
+    await db.disconnect();
+  });
+
+  describe('File format', () => {
+    let library: LibraryResponseDto;
+    beforeEach(async () => {
+      library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+        type: LibraryType.EXTERNAL,
+        name: 'Library',
+        importPaths: [`${__dirname}/../assets/formats/jpg`],
+        exclusionPatterns: [],
+      });
+    });
+
+    it('should import a jpg file', async () => {
+      library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
+        `${__dirname}/../assets/formats/jpg`,
+      ]);
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      await waitForQueues(jobService);
+
+      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+      expect(assets).toHaveLength(1);
+      console.log(assets);
+      const jpgAsset = assets[0];
+      expect(jpgAsset.type).toBe(AssetType.IMAGE);
+      expect(jpgAsset.originalFileName).toBe('el_torcal_rocks');
+      expect(jpgAsset.exifInfo?.exifImageHeight).toBe(341);
+      expect(jpgAsset.exifInfo?.exifImageWidth).toBe(512);
+    });
+
+    it('should import a jpeg file', async () => {
+      library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
+        `${__dirname}/../assets/formats/jpeg`,
+      ]);
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      await waitForQueues(jobService);
+
+      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+      expect(assets).toHaveLength(1);
+      console.log(assets);
+      const jpegAsset = assets[0];
+      expect(jpegAsset.type).toBe(AssetType.IMAGE);
+      expect(jpegAsset.originalFileName).toBe('el_torcal_rocks');
+      expect(jpegAsset.exifInfo?.exifImageHeight).toBe(341);
+      expect(jpegAsset.exifInfo?.exifImageWidth).toBe(512);
+    });
+
+    it('should import a heic file', async () => {
+      library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
+        `${__dirname}/../assets/formats/heic`,
+      ]);
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      await waitForQueues(jobService);
+
+      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+      expect(assets).toHaveLength(1);
+      const heicAsset = assets[0];
+      console.log(heicAsset);
+      expect(heicAsset.type).toBe(AssetType.IMAGE);
+      expect(heicAsset.originalFileName).toBe('IMG_2682');
+      expect(heicAsset.duration).toBe(null);
+      expect(heicAsset.fileCreatedAt).toBe('2029-03-21T11:04:00.000Z');
+    });
+  });
+});

server/test/e2e/formats2.e2e-spec.ts

@@ -0,0 +1,101 @@
+import { JobService, LibraryResponseDto, LoginResponseDto } from '@app/domain';
+import { AppModule } from '@app/immich/app.module';
+import { RedisIoAdapter } from '@app/infra';
+import { AssetType, LibraryType } from '@app/infra/entities';
+import { INestApplication, Logger } from '@nestjs/common';
+import { Test, TestingModule } from '@nestjs/testing';
+import { api } from '@test/api';
+import { waitForQueues } from '@test/test-utils';
+import { AppService as MicroAppService } from 'src/microservices/app.service';
+
+import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor';
+import { DockerComposeEnvironment } from 'testcontainers';
+
+describe('File format (e2e)', () => {
+  let app: INestApplication;
+  let jobService: JobService;
+  let server: any;
+  let moduleFixture: TestingModule;
+  let admin: LoginResponseDto;
+
+  beforeEach(async () => {
+    // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder
+
+    jest.useRealTimers();
+
+    await jobService.obliterateAll(true);
+
+    await api.authApi.adminSignUp(server);
+    admin = await api.authApi.adminLogin(server);
+    await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' });
+  });
+
+  describe('File format', () => {
+    let library: LibraryResponseDto;
+    beforeEach(async () => {
+      library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+        type: LibraryType.EXTERNAL,
+        name: 'Library',
+        importPaths: [`${__dirname}/../assets/formats/jpg`],
+        exclusionPatterns: [],
+      });
+    });
+
+    it('should import a jpg file', async () => {
+      library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
+        `${__dirname}/../assets/formats/jpg`,
+      ]);
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      await waitForQueues(jobService);
+
+      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+      expect(assets).toHaveLength(1);
+      console.log(assets);
+      const jpgAsset = assets[0];
+      expect(jpgAsset.type).toBe(AssetType.IMAGE);
+      expect(jpgAsset.originalFileName).toBe('el_torcal_rocks');
+      expect(jpgAsset.exifInfo?.exifImageHeight).toBe(341);
+      expect(jpgAsset.exifInfo?.exifImageWidth).toBe(512);
+    });
+
+    it('should import a jpeg file', async () => {
+      library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
+        `${__dirname}/../assets/formats/jpeg`,
+      ]);
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      await waitForQueues(jobService);
+
+      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+      expect(assets).toHaveLength(1);
+      console.log(assets);
+      const jpegAsset = assets[0];
+      expect(jpegAsset.type).toBe(AssetType.IMAGE);
+      expect(jpegAsset.originalFileName).toBe('el_torcal_rocks');
+      expect(jpegAsset.exifInfo?.exifImageHeight).toBe(341);
+      expect(jpegAsset.exifInfo?.exifImageWidth).toBe(512);
+    });
+
+    it('should import a heic file', async () => {
+      library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
+        `${__dirname}/../assets/formats/heic`,
+      ]);
+
+      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+      await waitForQueues(jobService);
+
+      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+      expect(assets).toHaveLength(1);
+      const heicAsset = assets[0];
+      console.log(heicAsset);
+      expect(heicAsset.type).toBe(AssetType.IMAGE);
+      expect(heicAsset.originalFileName).toBe('IMG_2682');
+      expect(heicAsset.duration).toBe(null);
+      expect(heicAsset.fileCreatedAt).toBe('2029-03-21T11:04:00.000Z');
+    });
+  });
+});

server/test/e2e/library2.e2e-spec.ts

@@ -0,0 +1,94 @@
+import { JobService, LoginResponseDto } from '@app/domain';
+import { AppModule } from '@app/immich/app.module';
+import { RedisIoAdapter } from '@app/infra';
+import { LibraryType } from '@app/infra/entities';
+import { INestApplication, Logger } from '@nestjs/common';
+import { Test, TestingModule } from '@nestjs/testing';
+import { api } from '@test/api';
+import { db } from '@test/db';
+import { waitForQueues } from '@test/test-utils';
+import { AppService as MicroAppService } from 'src/microservices/app.service';
+
+import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor';
+
+describe('Library queue e2e', () => {
+  let app: INestApplication;
+  let jobService: JobService;
+  let server: any;
+  let moduleFixture: TestingModule;
+  let admin: LoginResponseDto;
+
+  beforeEach(async () => {
+    await db.reset();
+
+    jest.useRealTimers();
+
+    moduleFixture = await Test.createTestingModule({
+      imports: [AppModule],
+      providers: [MetadataExtractionProcessor, MicroAppService],
+    })
+      .setLogger(new Logger())
+      .compile();
+
+    app = moduleFixture.createNestApplication();
+
+    app.useWebSocketAdapter(new RedisIoAdapter(app));
+
+    await app.init();
+    app.enableShutdownHooks();
+    server = app.getHttpServer();
+
+    jobService = moduleFixture.get(JobService);
+
+    await moduleFixture.get(MicroAppService).init(true);
+
+    // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder
+
+    await jobService.obliterateAll(true);
+
+    await api.authApi.adminSignUp(server);
+    admin = await api.authApi.adminLogin(server);
+    await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' });
+  });
+
+  afterEach(async () => {
+    await jobService.obliterateAll(true);
+    await app.close();
+    await moduleFixture.close();
+    await db.disconnect();
+  });
+
+  it('should scan the whole folder', async () => {
+    const library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+      type: LibraryType.EXTERNAL,
+      name: 'Library',
+      importPaths: [`${__dirname}/../assets/nature`],
+      exclusionPatterns: [],
+    });
+
+    await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+    await waitForQueues(jobService);
+
+    const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+    expect(assets).toHaveLength(7);
+  });
+
+  it('scan with exclusions', async () => {
+    const library = await api.libraryApi.createLibrary(server, admin.accessToken, {
+      type: LibraryType.EXTERNAL,
+      name: 'Library',
+      importPaths: [`${__dirname}/../assets/nature/`],
+      exclusionPatterns: ['**/*o*/**', '**/*c*/**'],
+    });
+
+    await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {});
+
+    await waitForQueues(jobService);
+
+    const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
+
+    expect(assets).toHaveLength(1);
+    expect(assets[0].originalFileName).toBe('silver_fir');
+  });
+});

server/test/e2e/server-info.e2e-spec.ts

@@ -81,16 +81,16 @@ describe(`${ServerInfoController.name} (e2e)`, () => {
       const { status, body } = await request(server).get('/server-info/features');
       expect(status).toBe(200);
       expect(body).toEqual({
-        clipEncode: true,
+        clipEncode: false,
         configFile: false,
-        facialRecognition: true,
+        facialRecognition: false,
         map: true,
         oauth: false,
         oauthAutoLaunch: false,
         passwordLogin: true,
         search: false,
         sidecar: true,
-        tagImage: true,
+        tagImage: false,
       });
     });
   });

server/test/e2e/setup.ts

@@ -1,8 +1,28 @@
 import { PostgreSqlContainer } from '@testcontainers/postgresql';
-import { GenericContainer } from 'testcontainers';
+import { DockerComposeEnvironment, GenericContainer, Wait } from 'testcontainers';
 export default async () => {
+  const composeFilePath = '../docker/';
+  const composeFile = 'docker-compose.e2e.yml';
+
+  process.env.IMMICH_SERVER_URL = 'asdf';
+  process.env.IMMICH_API_URL_EXTERNAL = 'asdf';
+  process.env.TYPESENSE_ENABLED = 'false';
+
+  process.env.IMMICH_MACHINE_LEARNING_ENABLED = 'false';
+
+  const environment = await new DockerComposeEnvironment(composeFilePath, composeFile)
+    .withBuild()
+    // .withWaitStrategy('immich_server-e2e', Wait.forLogMessage('Immich Server is listening on'))
+    // .withWaitStrategy('immich_microservices-e2e ', Wait.forLogMessage('Immich Microservices is listening on'))
+    //.withBuild()
+    .up();
+  console.log(environment);
+};
+
+export async function waitForQueues() {
   process.env.NODE_ENV = 'development';
   process.env.TYPESENSE_ENABLED = 'false';
+  process.env.IMMICH_MACHINE_LEARNING_ENABLED = 'false';
 
   const pg = await new PostgreSqlContainer('postgres')
     .withExposedPorts(5432)
@@ -18,4 +38,4 @@ export default async () => {
 
   process.env.REDIS_PORT = String(redis.getMappedPort(6379));
   process.env.REDIS_HOSTNAME = redis.getHost();
-};
+}

server/test/repositories/job.repository.mock.ts

@@ -10,5 +10,6 @@ export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
     queue: jest.fn().mockImplementation(() => Promise.resolve()),
     getQueueStatus: jest.fn(),
     getJobCounts: jest.fn(),
+    obliterate: jest.fn(),
   };
 };

server/test/test-utils.ts

@@ -4,6 +4,7 @@ import {
   AuthDeviceResponseDto,
   AuthUserDto,
   CreateUserDto,
+  JobService,
   LibraryResponseDto,
   LoginCredentialDto,
   LoginResponseDto,
@@ -49,6 +50,34 @@ export function getAuthUser(): AuthUserDto {
   };
 }
 
+export function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function waitForQueues(jobService: JobService) {
+  let isFinished = false;
+  // TODO: this shouldn't be a while loop
+  while (!isFinished) {
+    const jobStatus = await jobService.getAllJobsStatus();
+
+    let jobsActive = false;
+    Object.values(jobStatus).forEach((job) => {
+      if (job.queueStatus.isActive) {
+        jobsActive = true;
+      }
+      if (job.queueStatus.active > 0 || job.queueStatus.waiting > 0) {
+        jobsActive = true;
+      }
+    });
+
+    if (!jobsActive) {
+      isFinished = true;
+    }
+
+    await sleep(100);
+  }
+}
+
 export const api = {
   adminSignUp: async (server: any) => {
     const { status, body } = await request(server).post('/auth/admin-sign-up').send(adminSignupStub);