From c96e3becc3fa12514dfc7892394b74128dcd6900 Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Mon, 25 Sep 2023 11:25:13 +0200 Subject: [PATCH] fix --- docker/docker-compose.e2e.yml | 146 +++++++++++++++++++++++++++ server/test/api/library-api.ts | 9 ++ server/test/e2e/formats.e2e-spec.ts | 136 +++++++++++++------------ server/test/e2e/formats2.e2e-spec.ts | 101 ++++++++++++++++++ server/test/e2e/library2.e2e-spec.ts | 70 ++++++------- server/test/e2e/setup.ts | 23 ++++- 6 files changed, 378 insertions(+), 107 deletions(-) create mode 100644 docker/docker-compose.e2e.yml create mode 100644 server/test/e2e/formats2.e2e-spec.ts diff --git a/docker/docker-compose.e2e.yml b/docker/docker-compose.e2e.yml new file mode 100644 index 000000000..70cd20f68 --- /dev/null +++ b/docker/docker-compose.e2e.yml @@ -0,0 +1,146 @@ +version: "3.8" + +services: + immich-server: + container_name: immich_server-e2e + image: immich-server-dev:latest + build: + context: ../server + dockerfile: Dockerfile + target: builder + command: npm run start:debug immich + volumes: + - ../server:/usr/src/app + - ${UPLOAD_LOCATION}:/usr/src/app/upload + - /usr/src/app/node_modules + ports: + - 3001:3001 + - 9230:9230 + env_file: + - .env + environment: + - NODE_ENV=development + depends_on: + - redis + - database + - typesense + + immich-machine-learning: + container_name: immich_machine_learning-e2e + image: immich-machine-learning-dev:latest + build: + context: ../machine-learning + dockerfile: Dockerfile + ports: + - 3003:3003 + volumes: + - ../machine-learning:/usr/src/app + - model-cache:/cache + env_file: + - .env + environment: + - NODE_ENV=development + depends_on: + - database + restart: unless-stopped + + immich-microservices: + container_name: immich_microservices-e2e + image: immich-microservices:latest + # extends: + # file: hwaccel.yml + # service: hwaccel + build: + context: ../server + dockerfile: Dockerfile + target: builder + command: npm run start:debug microservices + volumes: + - ../server:/usr/src/app + - ${UPLOAD_LOCATION}:/usr/src/app/upload + - /usr/src/app/node_modules + env_file: + - .env + ports: + - 9231:9230 + environment: + - NODE_ENV=development + depends_on: + - database + - immich-server + - typesense + + immich-web: + container_name: immich_web-e2e + image: immich-web-dev:1.9.0 + build: + context: ../web + dockerfile: Dockerfile + target: dev + command: npm run dev --host + env_file: + - .env + environment: + # Rename these values for svelte public interface + - PUBLIC_IMMICH_SERVER_URL=${IMMICH_SERVER_URL} + - PUBLIC_IMMICH_API_URL_EXTERNAL=${IMMICH_API_URL_EXTERNAL} + ports: + - 3000:3000 + - 24678:24678 + volumes: + - ../web:/usr/src/app + - /usr/src/app/node_modules + restart: unless-stopped + depends_on: + - immich-server + + typesense: + container_name: immich_typesense-e2e + image: typesense/typesense:0.24.1@sha256:9bcff2b829f12074426ca044b56160ca9d777a0c488303469143dd9f8259d4dd + environment: + - TYPESENSE_API_KEY=${TYPESENSE_API_KEY} + - TYPESENSE_DATA_DIR=/data + # remove this to get debug messages + - GLOG_minloglevel=1 + volumes: + - tsdata:/data + + redis: + container_name: immich_redis-e2e + image: redis:6.2-alpine@sha256:70a7a5b641117670beae0d80658430853896b5ef269ccf00d1827427e3263fa3 + + database: + container_name: immich_postgres-e2e + image: postgres:14-alpine@sha256:28407a9961e76f2d285dc6991e8e48893503cc3836a4755bbc2d40bcc272a441 + env_file: + - .env + environment: + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_USER: ${DB_USERNAME} + POSTGRES_DB: ${DB_DATABASE_NAME} + volumes: + - pgdata:/var/lib/postgresql/data + ports: + - 5432:5432 + + immich-proxy: + container_name: immich_proxy-e2e + image: immich-proxy-dev:latest + environment: + # Make sure these values get passed through from the env file + - IMMICH_SERVER_URL + - IMMICH_WEB_URL + build: + context: ../nginx + dockerfile: Dockerfile + ports: + - 2285:8080 + depends_on: + - immich-server + - immich-web + restart: unless-stopped + +volumes: + pgdata: + model-cache: + tsdata: diff --git a/server/test/api/library-api.ts b/server/test/api/library-api.ts index a4d219e12..2682c4333 100644 --- a/server/test/api/library-api.ts +++ b/server/test/api/library-api.ts @@ -17,6 +17,15 @@ export const libraryApi = { return body as LibraryResponseDto; }, + setImportPaths: async (server: any, accessToken: string, id: string, importPaths: string[]) => { + const { body, status } = await request(server) + .put(`/library/${id}`) + .set('Authorization', `Bearer ${accessToken}`) + .send({ importPaths }); + expect(status).toBe(200); + return body as LibraryResponseDto; + }, + scanLibrary: async (server: any, accessToken: string, id: string, dto: ScanLibraryDto) => { const { status } = await request(server) .post(`/library/${id}/scan`) diff --git a/server/test/e2e/formats.e2e-spec.ts b/server/test/e2e/formats.e2e-spec.ts index 850001831..8b44fddfb 100644 --- a/server/test/e2e/formats.e2e-spec.ts +++ b/server/test/e2e/formats.e2e-spec.ts @@ -1,7 +1,7 @@ -import { JobService, LoginResponseDto } from '@app/domain'; +import { JobService, LibraryResponseDto, LoginResponseDto } from '@app/domain'; import { AppModule } from '@app/immich/app.module'; import { RedisIoAdapter } from '@app/infra'; -import { LibraryType } from '@app/infra/entities'; +import { AssetType, LibraryType } from '@app/infra/entities'; import { INestApplication, Logger } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; import { api } from '@test/api'; @@ -18,7 +18,9 @@ describe('File format (e2e)', () => { let moduleFixture: TestingModule; let admin: LoginResponseDto; - beforeAll(async () => { + beforeEach(async () => { + // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder + await db.reset(); jest.useRealTimers(); @@ -27,7 +29,7 @@ describe('File format (e2e)', () => { imports: [AppModule], providers: [MetadataExtractionProcessor, MicroAppService], }) - .setLogger(new Logger()) + // .setLogger(new Logger()) .compile(); app = moduleFixture.createNestApplication(); @@ -41,21 +43,6 @@ describe('File format (e2e)', () => { jobService = moduleFixture.get(JobService); await moduleFixture.get(MicroAppService).init(true); - }); - - afterAll(async () => { - console.log(await jobService.getAllJobsStatus()); - await jobService.obliterateAll(true); - await app.close(); - await moduleFixture.close(); - await db.disconnect(); - }); - - beforeEach(async () => { - // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder - - await db.reset(); - await jobService.obliterateAll(true); await api.authApi.adminSignUp(server); @@ -63,64 +50,79 @@ describe('File format (e2e)', () => { await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' }); }); - it('should import a jpg file', async () => { - const library = await api.libraryApi.createLibrary(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - name: 'Library', - importPaths: [`${__dirname}/../assets/formats/jpg`], - exclusionPatterns: [], - }); - - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); - - await waitForQueues(jobService); - - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - expect(assets).toHaveLength(1); - console.log(assets); - const jpgAsset = assets[0]; - expect(jpgAsset.type).toBe('image/jpeg'); - expect(jpgAsset.originalFileName).toBe('el_torcal_rocks'); - expect(jpgAsset.exifInfo?.exifImageHeight).toBe(341); - expect(jpgAsset.exifInfo?.exifImageWidth).toBe(512); + afterEach(async () => { + await jobService.obliterateAll(true); + await app.close(); + await moduleFixture.close(); + await db.disconnect(); }); - it('should import a jpeg file', async () => { - const library = await api.libraryApi.createLibrary(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - name: 'Library', - importPaths: [`${__dirname}/../assets/formats/jpeg`], - exclusionPatterns: [], + describe('File format', () => { + let library: LibraryResponseDto; + beforeEach(async () => { + library = await api.libraryApi.createLibrary(server, admin.accessToken, { + type: LibraryType.EXTERNAL, + name: 'Library', + importPaths: [`${__dirname}/../assets/formats/jpg`], + exclusionPatterns: [], + }); }); - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + it('should import a jpg file', async () => { + library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [ + `${__dirname}/../assets/formats/jpg`, + ]); - await waitForQueues(jobService); + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - expect(assets).toHaveLength(1); - console.log(assets); - }); + await waitForQueues(jobService); - it('should import a heic file', async () => { - const library = await api.libraryApi.createLibrary(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - name: 'Library', - importPaths: [`${__dirname}/../assets/formats/heic`], - exclusionPatterns: [], + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + console.log(assets); + const jpgAsset = assets[0]; + expect(jpgAsset.type).toBe(AssetType.IMAGE); + expect(jpgAsset.originalFileName).toBe('el_torcal_rocks'); + expect(jpgAsset.exifInfo?.exifImageHeight).toBe(341); + expect(jpgAsset.exifInfo?.exifImageWidth).toBe(512); }); - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + it('should import a jpeg file', async () => { + library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [ + `${__dirname}/../assets/formats/jpeg`, + ]); - await waitForQueues(jobService); + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - expect(assets).toHaveLength(1); - const heicAsset = assets[0]; - console.log(heicAsset); - expect(heicAsset.type).toBe('image/heic'); - expect(heicAsset.originalFileName).toBe('IMG_2682'); - expect(heicAsset.duration).toBe(null); - expect(heicAsset.fileCreatedAt).toBe('2029-03-21T11:04:00.000Z'); + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + console.log(assets); + const jpegAsset = assets[0]; + expect(jpegAsset.type).toBe(AssetType.IMAGE); + expect(jpegAsset.originalFileName).toBe('el_torcal_rocks'); + expect(jpegAsset.exifInfo?.exifImageHeight).toBe(341); + expect(jpegAsset.exifInfo?.exifImageWidth).toBe(512); + }); + + it('should import a heic file', async () => { + library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [ + `${__dirname}/../assets/formats/heic`, + ]); + + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + const heicAsset = assets[0]; + console.log(heicAsset); + expect(heicAsset.type).toBe(AssetType.IMAGE); + expect(heicAsset.originalFileName).toBe('IMG_2682'); + expect(heicAsset.duration).toBe(null); + expect(heicAsset.fileCreatedAt).toBe('2029-03-21T11:04:00.000Z'); + }); }); }); diff --git a/server/test/e2e/formats2.e2e-spec.ts b/server/test/e2e/formats2.e2e-spec.ts new file mode 100644 index 000000000..c793bee9c --- /dev/null +++ b/server/test/e2e/formats2.e2e-spec.ts @@ -0,0 +1,101 @@ +import { JobService, LibraryResponseDto, LoginResponseDto } from '@app/domain'; +import { AppModule } from '@app/immich/app.module'; +import { RedisIoAdapter } from '@app/infra'; +import { AssetType, LibraryType } from '@app/infra/entities'; +import { INestApplication, Logger } from '@nestjs/common'; +import { Test, TestingModule } from '@nestjs/testing'; +import { api } from '@test/api'; +import { waitForQueues } from '@test/test-utils'; +import { AppService as MicroAppService } from 'src/microservices/app.service'; + +import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor'; +import { DockerComposeEnvironment } from 'testcontainers'; + +describe('File format (e2e)', () => { + let app: INestApplication; + let jobService: JobService; + let server: any; + let moduleFixture: TestingModule; + let admin: LoginResponseDto; + + beforeEach(async () => { + // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder + + jest.useRealTimers(); + + await jobService.obliterateAll(true); + + await api.authApi.adminSignUp(server); + admin = await api.authApi.adminLogin(server); + await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' }); + }); + + describe('File format', () => { + let library: LibraryResponseDto; + beforeEach(async () => { + library = await api.libraryApi.createLibrary(server, admin.accessToken, { + type: LibraryType.EXTERNAL, + name: 'Library', + importPaths: [`${__dirname}/../assets/formats/jpg`], + exclusionPatterns: [], + }); + }); + + it('should import a jpg file', async () => { + library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [ + `${__dirname}/../assets/formats/jpg`, + ]); + + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + console.log(assets); + const jpgAsset = assets[0]; + expect(jpgAsset.type).toBe(AssetType.IMAGE); + expect(jpgAsset.originalFileName).toBe('el_torcal_rocks'); + expect(jpgAsset.exifInfo?.exifImageHeight).toBe(341); + expect(jpgAsset.exifInfo?.exifImageWidth).toBe(512); + }); + + it('should import a jpeg file', async () => { + library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [ + `${__dirname}/../assets/formats/jpeg`, + ]); + + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + console.log(assets); + const jpegAsset = assets[0]; + expect(jpegAsset.type).toBe(AssetType.IMAGE); + expect(jpegAsset.originalFileName).toBe('el_torcal_rocks'); + expect(jpegAsset.exifInfo?.exifImageHeight).toBe(341); + expect(jpegAsset.exifInfo?.exifImageWidth).toBe(512); + }); + + it('should import a heic file', async () => { + library = await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [ + `${__dirname}/../assets/formats/heic`, + ]); + + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + const heicAsset = assets[0]; + console.log(heicAsset); + expect(heicAsset.type).toBe(AssetType.IMAGE); + expect(heicAsset.originalFileName).toBe('IMG_2682'); + expect(heicAsset.duration).toBe(null); + expect(heicAsset.fileCreatedAt).toBe('2029-03-21T11:04:00.000Z'); + }); + }); +}); diff --git a/server/test/e2e/library2.e2e-spec.ts b/server/test/e2e/library2.e2e-spec.ts index 4c5a79c21..856039e71 100644 --- a/server/test/e2e/library2.e2e-spec.ts +++ b/server/test/e2e/library2.e2e-spec.ts @@ -18,7 +18,7 @@ describe('Library queue e2e', () => { let moduleFixture: TestingModule; let admin: LoginResponseDto; - beforeAll(async () => { + beforeEach(async () => { await db.reset(); jest.useRealTimers(); @@ -41,60 +41,54 @@ describe('Library queue e2e', () => { jobService = moduleFixture.get(JobService); await moduleFixture.get(MicroAppService).init(true); + + // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder + + await jobService.obliterateAll(true); + + await api.authApi.adminSignUp(server); + admin = await api.authApi.adminLogin(server); + await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' }); }); - afterAll(async () => { + afterEach(async () => { await jobService.obliterateAll(true); await app.close(); await moduleFixture.close(); await db.disconnect(); }); - describe('can import library', () => { - beforeEach(async () => { - // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder - - await db.reset(); - - await jobService.obliterateAll(true); - - await api.authApi.adminSignUp(server); - admin = await api.authApi.adminLogin(server); - await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' }); + it('should scan the whole folder', async () => { + const library = await api.libraryApi.createLibrary(server, admin.accessToken, { + type: LibraryType.EXTERNAL, + name: 'Library', + importPaths: [`${__dirname}/../assets/nature`], + exclusionPatterns: [], }); - it('should scan the whole folder', async () => { - const library = await api.libraryApi.createLibrary(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - name: 'Library', - importPaths: [`${__dirname}/../assets/nature`], - exclusionPatterns: [], - }); + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + await waitForQueues(jobService); - await waitForQueues(jobService); + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(7); + }); - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - expect(assets).toHaveLength(7); + it('scan with exclusions', async () => { + const library = await api.libraryApi.createLibrary(server, admin.accessToken, { + type: LibraryType.EXTERNAL, + name: 'Library', + importPaths: [`${__dirname}/../assets/nature/`], + exclusionPatterns: ['**/*o*/**', '**/*c*/**'], }); - it('scan with exclusions', async () => { - const library = await api.libraryApi.createLibrary(server, admin.accessToken, { - type: LibraryType.EXTERNAL, - name: 'Library', - importPaths: [`${__dirname}/../assets/nature/`], - exclusionPatterns: ['**/*o*/**', '**/*c*/**'], - }); + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + await waitForQueues(jobService); - await waitForQueues(jobService); + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - const assets = await api.assetApi.getAllAssets(server, admin.accessToken); - - expect(assets).toHaveLength(1); - expect(assets[0].originalFileName).toBe('silver_fir'); - }); + expect(assets).toHaveLength(1); + expect(assets[0].originalFileName).toBe('silver_fir'); }); }); diff --git a/server/test/e2e/setup.ts b/server/test/e2e/setup.ts index c84145ac9..b41aee23c 100644 --- a/server/test/e2e/setup.ts +++ b/server/test/e2e/setup.ts @@ -1,6 +1,25 @@ import { PostgreSqlContainer } from '@testcontainers/postgresql'; -import { GenericContainer } from 'testcontainers'; +import { DockerComposeEnvironment, GenericContainer, Wait } from 'testcontainers'; export default async () => { + const composeFilePath = '../docker/'; + const composeFile = 'docker-compose.e2e.yml'; + + process.env.IMMICH_SERVER_URL = 'asdf'; + process.env.IMMICH_API_URL_EXTERNAL = 'asdf'; + process.env.TYPESENSE_ENABLED = 'false'; + + process.env.IMMICH_MACHINE_LEARNING_ENABLED = 'false'; + + const environment = await new DockerComposeEnvironment(composeFilePath, composeFile) + .withBuild() + // .withWaitStrategy('immich_server-e2e', Wait.forLogMessage('Immich Server is listening on')) + // .withWaitStrategy('immich_microservices-e2e ', Wait.forLogMessage('Immich Microservices is listening on')) + //.withBuild() + .up(); + console.log(environment); +}; + +export async function waitForQueues() { process.env.NODE_ENV = 'development'; process.env.TYPESENSE_ENABLED = 'false'; process.env.IMMICH_MACHINE_LEARNING_ENABLED = 'false'; @@ -19,4 +38,4 @@ export default async () => { process.env.REDIS_PORT = String(redis.getMappedPort(6379)); process.env.REDIS_HOSTNAME = redis.getHost(); -}; +}