From a094805c275be2dc922ceb4f7ec017d5282404fd Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Fri, 22 Sep 2023 13:28:10 +0200 Subject: [PATCH] install docker dependencies --- .github/workflows/test.yml | 12 ++- server/package.json | 2 +- server/src/domain/job/job.repository.ts | 1 - server/src/domain/job/job.service.ts | 4 - server/src/domain/library/library.service.ts | 2 - .../src/infra/repositories/job.repository.ts | 10 -- server/src/microservices/app.service.ts | 2 + .../metadata-extraction.processor.ts | 5 +- server/test/e2e/formats.e2e-spec.ts | 93 +++++++++++++++++++ server/test/e2e/library2.e2e-spec.ts | 64 ++++++------- .../test/repositories/job.repository.mock.ts | 1 - server/test/test-utils.ts | 25 +++++ 12 files changed, 160 insertions(+), 61 deletions(-) create mode 100644 server/test/e2e/formats.e2e-spec.ts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2aad0b93e..1cce04f3d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,10 @@ jobs: run: working-directory: ./server + env: + ENV LD_LIBRARY_PATH: /usr/local/lib:$LD_LIBRARY_PATH + ENV LD_RUN_PATH: /usr/local/lib:$LD_RUN_PATH + steps: - name: Checkout code uses: actions/checkout@v4 @@ -30,6 +34,12 @@ jobs: with: node-version: "21.0.0-nightly20230921480ab8c3a4" + - name: Install dependencies + run: sudo apt-get update && apt-get install -yqq build-essential ninja-build meson pkg-config jq zlib1g autoconf libglib2.0-dev libexpat1-dev librsvg2-dev libexif-dev libwebp-dev liborc-0.4-dev libjpeg62-turbo-dev libgsf-1-dev libspng-dev libjxl-dev libheif-dev liblcms2-2 mesa-va-drivers libmimalloc2.0 $(if [ $(arch) = "x86_64" ]; then echo "intel-media-va-driver-non-free"; fi) && ./install-ffmpeg.sh && apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* + + - name: Install libraw, imagemagick, libvips + run: ./bin/build-libraw.sh && ./bin/build-imagemagick.sh && ./bin/build-libvips.sh + - name: Checkout test assets uses: actions/checkout@v4 with: @@ -37,7 +47,7 @@ jobs: path: ./server/test/assets - name: Run e2e tests - run: NODE_OPTIONS='--experimental-vm-modules' npm run test:e2e + run: npm run test:e2e -- --forceExit if: ${{ !cancelled() }} doc-tests: diff --git a/server/package.json b/server/package.json index 38426238d..d056050bf 100644 --- a/server/package.json +++ b/server/package.json @@ -26,7 +26,7 @@ "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "jest --config test/e2e/jest-e2e.json --runInBand --detectOpenHandles --verbose", + "test:e2e": "NODE_OPTIONS='--experimental-vm-modules' jest --config test/e2e/jest-e2e.json --runInBand", "typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js", "typeorm:migrations:create": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:create", "typeorm:migrations:generate": "node --require ts-node/register ./node_modules/typeorm/cli.js migration:generate -d ./src/infra/database.config.ts", diff --git a/server/src/domain/job/job.repository.ts b/server/src/domain/job/job.repository.ts index 35b162963..726dffe4f 100644 --- a/server/src/domain/job/job.repository.ts +++ b/server/src/domain/job/job.repository.ts @@ -112,5 +112,4 @@ export interface IJobRepository { getQueueStatus(name: QueueName): Promise; getJobCounts(name: QueueName): Promise; obliterate(name: QueueName, force: boolean): Promise; - closeWorkers(): Promise; } diff --git a/server/src/domain/job/job.service.ts b/server/src/domain/job/job.service.ts index aae5f5e66..0573b0172 100644 --- a/server/src/domain/job/job.service.ts +++ b/server/src/domain/job/job.service.ts @@ -69,10 +69,6 @@ export class JobService { } } - async closeAll(): Promise { - await this.jobRepository.closeWorkers(); - } - private async start(name: QueueName, { force }: JobCommandDto): Promise { const { isActive } = await this.jobRepository.getQueueStatus(name); if (isActive) { diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index bffe85ef4..4cce99646 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -350,8 +350,6 @@ export class LibraryService { } async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise { - console.log('Handle queue asset refresh: ' + JSON.stringify(job)); - console.log(await this.repository.getAll(true, LibraryType.EXTERNAL)); const library = await this.repository.get(job.id); if (!library || library.type !== LibraryType.EXTERNAL) { this.logger.warn('Can only refresh external libraries'); diff --git a/server/src/infra/repositories/job.repository.ts b/server/src/infra/repositories/job.repository.ts index 9086a34ed..638fd70fd 100644 --- a/server/src/infra/repositories/job.repository.ts +++ b/server/src/infra/repositories/job.repository.ts @@ -28,16 +28,6 @@ export class JobRepository implements IJobRepository { worker.concurrency = concurrency; } - async closeWorkers() { - for (const queue in Object.keys(this.workers)) { - const queueName = queue as QueueName; - const worker = this.workers[queueName]; - if (worker) { - await worker.close(); - } - } - } - async getQueueStatus(name: QueueName): Promise { const queue = this.getQueue(name); diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index f8eed3a40..fbf53f33b 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -44,8 +44,10 @@ export class AppService { async init(clearBeforeStart = false) { if (clearBeforeStart) { + // Clear all jobs on application startup, mainly used for e2e testing await this.jobService.obliterateAll(true); } + await this.jobService.registerHandlers({ [JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data), [JobName.CLEAN_OLD_AUDIT_LOGS]: () => this.auditService.handleCleanup(), diff --git a/server/src/microservices/processors/metadata-extraction.processor.ts b/server/src/microservices/processors/metadata-extraction.processor.ts index a2fcaa37f..b0274cb57 100644 --- a/server/src/microservices/processors/metadata-extraction.processor.ts +++ b/server/src/microservices/processors/metadata-extraction.processor.ts @@ -76,10 +76,9 @@ export class MetadataExtractionProcessor { await this.geocodingRepository.deleteCache(); } this.logger.log('Initializing Reverse Geocoding'); + await this.jobRepository.pause(QueueName.METADATA_EXTRACTION); - - //await this.geocodingRepository.init(); - + await this.geocodingRepository.init(); await this.jobRepository.resume(QueueName.METADATA_EXTRACTION); this.logger.log('Reverse Geocoding Initialized'); diff --git a/server/test/e2e/formats.e2e-spec.ts b/server/test/e2e/formats.e2e-spec.ts new file mode 100644 index 000000000..3a7e61852 --- /dev/null +++ b/server/test/e2e/formats.e2e-spec.ts @@ -0,0 +1,93 @@ +import { JobService, LoginResponseDto, QueueName } from '@app/domain'; +import { AppModule } from '@app/immich/app.module'; +import { LibraryType } from '@app/infra/entities'; +import { INestApplication, Logger } from '@nestjs/common'; +import { Test, TestingModule } from '@nestjs/testing'; +import { api } from '@test/api'; +import { db } from '@test/db'; +import { waitForQueues } from '@test/test-utils'; +import { AppService as MicroAppService } from 'src/microservices/app.service'; + +import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor'; + +describe('File format (e2e)', () => { + let app: INestApplication; + let jobService: JobService; + let server: any; + let moduleFixture: TestingModule; + let admin: LoginResponseDto; + + beforeAll(async () => { + jest.useRealTimers(); + + moduleFixture = await Test.createTestingModule({ + imports: [AppModule], + providers: [MetadataExtractionProcessor, MicroAppService], + }) + .setLogger(new Logger()) + .compile(); + + app = moduleFixture.createNestApplication(); + + await app.init(); + app.enableShutdownHooks(); + server = app.getHttpServer(); + + jobService = moduleFixture.get(JobService); + + await moduleFixture.get(MicroAppService).init(true); + }); + + beforeEach(async () => { + // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder + + await db.reset(); + + await jobService.obliterateAll(true); + + await api.authApi.adminSignUp(server); + admin = await api.authApi.adminLogin(server); + await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' }); + }); + + it('should import a jpg file', async () => { + const library = await api.libraryApi.createLibrary(server, admin.accessToken, { + type: LibraryType.EXTERNAL, + name: 'Library', + importPaths: [`${__dirname}/../assets/formats/jpg`], + exclusionPatterns: [], + }); + + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + }); + + it('should import a heic file', async () => { + const library = await api.libraryApi.createLibrary(server, admin.accessToken, { + type: LibraryType.EXTERNAL, + name: 'Library', + importPaths: [`${__dirname}/../assets/formats/heic`], + exclusionPatterns: [], + }); + + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + expect(assets).toHaveLength(1); + console.log(assets); + }); + + afterAll(async () => { + console.log(await jobService.getAllJobsStatus()); + await jobService.obliterateAll(true); + await app.close(); + await moduleFixture.close(); + await db.disconnect(); + }); +}); diff --git a/server/test/e2e/library2.e2e-spec.ts b/server/test/e2e/library2.e2e-spec.ts index cf0f431e9..645e2a7ee 100644 --- a/server/test/e2e/library2.e2e-spec.ts +++ b/server/test/e2e/library2.e2e-spec.ts @@ -1,25 +1,20 @@ import { JobService, LoginResponseDto, QueueName } from '@app/domain'; import { AppModule } from '@app/immich/app.module'; import { LibraryType } from '@app/infra/entities'; -import { JobRepository } from '@app/infra/repositories'; import { INestApplication, Logger } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; import { api } from '@test/api'; import { db } from '@test/db'; -import { sleep } from '@test/test-utils'; +import { waitForQueues } from '@test/test-utils'; import { AppService as MicroAppService } from 'src/microservices/app.service'; import { MetadataExtractionProcessor } from 'src/microservices/processors/metadata-extraction.processor'; -describe('libe2e', () => { +describe('Library queue e2e', () => { let app: INestApplication; - let jobService: JobService; - let server: any; - let moduleFixture: TestingModule; - let admin: LoginResponseDto; beforeAll(async () => { @@ -44,7 +39,9 @@ describe('libe2e', () => { }); describe('can import library', () => { - beforeAll(async () => { + beforeEach(async () => { + // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder + await db.reset(); await jobService.obliterateAll(true); @@ -52,7 +49,9 @@ describe('libe2e', () => { await api.authApi.adminSignUp(server); admin = await api.authApi.adminLogin(server); await api.userApi.update(server, admin.accessToken, { id: admin.userId, externalPath: '/' }); + }); + it('should scan the whole folder', async () => { const library = await api.libraryApi.createLibrary(server, admin.accessToken, { type: LibraryType.EXTERNAL, name: 'Library', @@ -60,45 +59,34 @@ describe('libe2e', () => { exclusionPatterns: [], }); - console.log(await api.libraryApi.getAll(server, admin.accessToken)); - - // We expect https://github.com/etnoy/immich-test-assets to be cloned into the e2e/assets folder - await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); - let isFinished = false; - // TODO: this shouldn't be a while loop - while (!isFinished) { - const jobStatus = await api.jobApi.getAllJobsStatus(server, admin.accessToken); + await waitForQueues(jobService); - let jobsActive = false; - Object.values(jobStatus).forEach((job) => { - if (job.queueStatus.isActive) { - jobsActive = true; - } - if (job.queueStatus.active > 0 || job.queueStatus.waiting > 0) { - jobsActive = true; - } - }); - - if (!jobsActive) { - isFinished = true; - } - - await sleep(200); - } - }); - - it('scans the library', async () => { const assets = await api.assetApi.getAllAssets(server, admin.accessToken); expect(assets).toHaveLength(7); }); + + it('scan with exclusions', async () => { + const library = await api.libraryApi.createLibrary(server, admin.accessToken, { + type: LibraryType.EXTERNAL, + name: 'Library', + importPaths: [`${__dirname}/../assets/nature/`], + exclusionPatterns: ['**/*o*/**', '**/*c*/**'], + }); + + await api.libraryApi.scanLibrary(server, admin.accessToken, library.id, {}); + + await waitForQueues(jobService); + + const assets = await api.assetApi.getAllAssets(server, admin.accessToken); + + expect(assets).toHaveLength(1); + expect(assets[0].originalFileName).toBe('silver_fir'); + }); }); - afterEach(async () => {}); - afterAll(async () => { - await jobService.closeAll(); await jobService.obliterateAll(true); await app.close(); await moduleFixture.close(); diff --git a/server/test/repositories/job.repository.mock.ts b/server/test/repositories/job.repository.mock.ts index 1dc30c15f..47b1750a5 100644 --- a/server/test/repositories/job.repository.mock.ts +++ b/server/test/repositories/job.repository.mock.ts @@ -11,6 +11,5 @@ export const newJobRepositoryMock = (): jest.Mocked => { getQueueStatus: jest.fn(), getJobCounts: jest.fn(), obliterate: jest.fn(), - closeWorkers: jest.fn(), }; }; diff --git a/server/test/test-utils.ts b/server/test/test-utils.ts index 42fcd9f1c..da1a71769 100644 --- a/server/test/test-utils.ts +++ b/server/test/test-utils.ts @@ -4,6 +4,7 @@ import { AuthDeviceResponseDto, AuthUserDto, CreateUserDto, + JobService, LibraryResponseDto, LoginCredentialDto, LoginResponseDto, @@ -53,6 +54,30 @@ export function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } +export async function waitForQueues(jobService: JobService) { + let isFinished = false; + // TODO: this shouldn't be a while loop + while (!isFinished) { + const jobStatus = await jobService.getAllJobsStatus(); + + let jobsActive = false; + Object.values(jobStatus).forEach((job) => { + if (job.queueStatus.isActive) { + jobsActive = true; + } + if (job.queueStatus.active > 0 || job.queueStatus.waiting > 0) { + jobsActive = true; + } + }); + + if (!jobsActive) { + isFinished = true; + } + + await sleep(100); + } +} + export const api = { adminSignUp: async (server: any) => { const { status, body } = await request(server).post('/auth/admin-sign-up').send(adminSignupStub);