From 71388eaec235f6a772486b3e8ae5cee8a8c2ccde Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Sat, 11 Nov 2023 18:59:46 -0500 Subject: [PATCH] use pgvecto.rs --- docker/docker-compose.dev.yml | 3 +- docker/docker-compose.prod.yml | 3 +- docker/docker-compose.yml | 3 +- server/package-lock.json | 22 ++ server/package.json | 1 + .../domain/repositories/person.repository.ts | 4 +- .../repositories/smart-info.repository.ts | 4 +- .../domain/smart-info/smart-info.service.ts | 2 +- server/src/infra/entities/asset.entity.ts | 4 + server/src/infra/entities/index.ts | 3 + .../src/infra/entities/smart-info.entity.ts | 7 - .../src/infra/entities/smart-search.entity.ts | 18 + .../migrations/1693228677355-UsePgVector.ts | 32 -- ...6875736010-AddSmartInfoTextSearchColumn.ts | 30 -- ...76192604-CreateSmartInfoTextSearchIndex.ts | 22 +- .../1699419684990-AddExifCityIndex.ts | 13 - .../1699419700539-AddSmartInfoTagsIndex.ts | 13 - .../1699586761207-AddCLIPEmbeddingIndex.ts | 19 - .../migrations/1699746198141-UsePgVector.ts | 47 +++ .../1699746301742-AddCLIPEmbeddingIndex.ts | 20 + .../1699746316571-AddExifCityIndex.ts | 13 + .../1699746328479-AddSmartInfoTagsIndex.ts | 13 + .../1699746444644-AddFaceEmbeddingIndex.ts | 20 + .../infra/repositories/asset.repository.ts | 6 +- .../repositories/smart-info.repository.ts | 76 ++-- web/package-lock.json | 352 ++++++++++++++++++ web/package.json | 1 + 27 files changed, 586 insertions(+), 165 deletions(-) create mode 100644 server/src/infra/entities/smart-search.entity.ts delete mode 100644 server/src/infra/migrations/1693228677355-UsePgVector.ts delete mode 100644 server/src/infra/migrations/1696875736010-AddSmartInfoTextSearchColumn.ts delete mode 100644 server/src/infra/migrations/1699419684990-AddExifCityIndex.ts delete mode 100644 server/src/infra/migrations/1699419700539-AddSmartInfoTagsIndex.ts delete mode 100644 server/src/infra/migrations/1699586761207-AddCLIPEmbeddingIndex.ts create mode 100644 server/src/infra/migrations/1699746198141-UsePgVector.ts create mode 100644 server/src/infra/migrations/1699746301742-AddCLIPEmbeddingIndex.ts create mode 100644 server/src/infra/migrations/1699746316571-AddExifCityIndex.ts create mode 100644 server/src/infra/migrations/1699746328479-AddSmartInfoTagsIndex.ts create mode 100644 server/src/infra/migrations/1699746444644-AddFaceEmbeddingIndex.ts diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index eae02763a..ed720c50a 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -99,8 +99,7 @@ services: database: container_name: immich_postgres - # image: postgres:14-alpine@sha256:28407a9961e76f2d285dc6991e8e48893503cc3836a4755bbc2d40bcc272a441 - image: ankane/pgvector + image: tensorchord/pgvecto-rs env_file: - .env environment: diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 83ccdcda0..11421654b 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -57,8 +57,7 @@ services: database: container_name: immich_postgres - # image: postgres:14-alpine@sha256:28407a9961e76f2d285dc6991e8e48893503cc3836a4755bbc2d40bcc272a441 - image: ankane/pgvector + image: tensorchord/pgvecto-rs env_file: - .env environment: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index cdae6e6da..d529e2d23 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -60,8 +60,7 @@ services: database: container_name: immich_postgres - # image: postgres:14-alpine@sha256:28407a9961e76f2d285dc6991e8e48893503cc3836a4755bbc2d40bcc272a441 - image: ankane/pgvector + image: tensorchord/pgvecto-rs env_file: - .env environment: diff --git a/server/package-lock.json b/server/package-lock.json index 05ab60984..1fdc488ed 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -54,6 +54,7 @@ "ua-parser-js": "^1.0.35" }, "devDependencies": { + "@esbuild/linux-x64": "^0.19.5", "@nestjs/cli": "^10.1.16", "@nestjs/schematics": "^10.0.2", "@nestjs/testing": "^10.2.2", @@ -914,6 +915,21 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@esbuild/linux-x64": { + "version": "0.19.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.5.tgz", + "integrity": "sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A==", + "cpu": [ + "x64" + ], + "dev": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -13390,6 +13406,12 @@ } } }, + "@esbuild/linux-x64": { + "version": "0.19.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.5.tgz", + "integrity": "sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A==", + "dev": true + }, "@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", diff --git a/server/package.json b/server/package.json index 553e25511..219e5d8f2 100644 --- a/server/package.json +++ b/server/package.json @@ -81,6 +81,7 @@ "ua-parser-js": "^1.0.35" }, "devDependencies": { + "@esbuild/linux-x64": "^0.19.5", "@nestjs/cli": "^10.1.16", "@nestjs/schematics": "^10.0.2", "@nestjs/testing": "^10.2.2", diff --git a/server/src/domain/repositories/person.repository.ts b/server/src/domain/repositories/person.repository.ts index 432362d3b..676b4e6ee 100644 --- a/server/src/domain/repositories/person.repository.ts +++ b/server/src/domain/repositories/person.repository.ts @@ -25,9 +25,11 @@ export interface PersonStatistics { assets: number; } +export type Embedding = number[]; + export interface EmbeddingSearch { ownerId: string; - embedding: number[]; + embedding: Embedding; numResults: number; maxDistance?: number; } diff --git a/server/src/domain/repositories/smart-info.repository.ts b/server/src/domain/repositories/smart-info.repository.ts index 0e5aa6983..533aecf1f 100644 --- a/server/src/domain/repositories/smart-info.repository.ts +++ b/server/src/domain/repositories/smart-info.repository.ts @@ -1,9 +1,9 @@ import { AssetEntity, SmartInfoEntity } from '@app/infra/entities'; -import { EmbeddingSearch } from '../repositories'; +import { Embedding, EmbeddingSearch } from '../repositories'; export const ISmartInfoRepository = 'ISmartInfoRepository'; export interface ISmartInfoRepository { searchByEmbedding(search: EmbeddingSearch): Promise; - upsert(info: Partial): Promise; + upsert(smartInfo: Partial, embedding?: Embedding): Promise; } diff --git a/server/src/domain/smart-info/smart-info.service.ts b/server/src/domain/smart-info/smart-info.service.ts index 743e23b9f..8af3bc7b7 100644 --- a/server/src/domain/smart-info/smart-info.service.ts +++ b/server/src/domain/smart-info/smart-info.service.ts @@ -105,7 +105,7 @@ export class SmartInfoService { machineLearning.clip, ); - await this.repository.upsert({ assetId: asset.id, clipEmbedding: clipEmbedding }); + await this.repository.upsert({ assetId: asset.id }, clipEmbedding); return true; } diff --git a/server/src/infra/entities/asset.entity.ts b/server/src/infra/entities/asset.entity.ts index b1f254da4..247903209 100644 --- a/server/src/infra/entities/asset.entity.ts +++ b/server/src/infra/entities/asset.entity.ts @@ -22,6 +22,7 @@ import { SharedLinkEntity } from './shared-link.entity'; import { SmartInfoEntity } from './smart-info.entity'; import { TagEntity } from './tag.entity'; import { UserEntity } from './user.entity'; +import { SmartSearchEntity } from '.'; export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum'; @@ -137,6 +138,9 @@ export class AssetEntity { @OneToOne(() => SmartInfoEntity, (smartInfoEntity) => smartInfoEntity.asset) smartInfo?: SmartInfoEntity; + @OneToOne(() => SmartSearchEntity, (smartSearchEntity) => smartSearchEntity.asset) + smartSearch?: SmartSearchEntity; + @ManyToMany(() => TagEntity, (tag) => tag.assets, { cascade: true }) @JoinTable({ name: 'tag_asset' }) tags!: TagEntity[]; diff --git a/server/src/infra/entities/index.ts b/server/src/infra/entities/index.ts index 6c662a20a..c4252b655 100644 --- a/server/src/infra/entities/index.ts +++ b/server/src/infra/entities/index.ts @@ -15,6 +15,7 @@ import { PartnerEntity } from './partner.entity'; import { PersonEntity } from './person.entity'; import { SharedLinkEntity } from './shared-link.entity'; import { SmartInfoEntity } from './smart-info.entity'; +import { SmartSearchEntity } from './smart-search.entity'; import { SystemConfigEntity } from './system-config.entity'; import { SystemMetadataEntity } from './system-metadata.entity'; import { TagEntity } from './tag.entity'; @@ -38,6 +39,7 @@ export * from './partner.entity'; export * from './person.entity'; export * from './shared-link.entity'; export * from './smart-info.entity'; +export * from './smart-search.entity'; export * from './system-config.entity'; export * from './system-metadata.entity'; export * from './tag.entity'; @@ -61,6 +63,7 @@ export const databaseEntities = [ PersonEntity, SharedLinkEntity, SmartInfoEntity, + SmartSearchEntity, SystemConfigEntity, SystemMetadataEntity, TagEntity, diff --git a/server/src/infra/entities/smart-info.entity.ts b/server/src/infra/entities/smart-info.entity.ts index cd6b9e172..ae3edd840 100644 --- a/server/src/infra/entities/smart-info.entity.ts +++ b/server/src/infra/entities/smart-info.entity.ts @@ -15,11 +15,4 @@ export class SmartInfoEntity { @Column({ type: 'text', array: true, nullable: true }) objects!: string[] | null; - - @Column({ - type: 'float4', - array: true, - nullable: true, - }) - clipEmbedding!: number[] | null; } diff --git a/server/src/infra/entities/smart-search.entity.ts b/server/src/infra/entities/smart-search.entity.ts new file mode 100644 index 000000000..bd9721441 --- /dev/null +++ b/server/src/infra/entities/smart-search.entity.ts @@ -0,0 +1,18 @@ +import { Column, Entity, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm'; +import { AssetEntity } from './asset.entity'; + +@Entity('smart_search') +export class SmartSearchEntity { + @OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true }) + @JoinColumn({ name: 'assetId', referencedColumnName: 'id' }) + asset?: AssetEntity; + + @PrimaryColumn() + assetId!: string; + + @Column({ + type: 'float4', + array: true, + }) + embedding!: number[]; +} diff --git a/server/src/infra/migrations/1693228677355-UsePgVector.ts b/server/src/infra/migrations/1693228677355-UsePgVector.ts deleted file mode 100644 index 2e42b6cfe..000000000 --- a/server/src/infra/migrations/1693228677355-UsePgVector.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class UsePgVector1693228677355 implements MigrationInterface { - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query('CREATE EXTENSION IF NOT EXISTS vector'); - - const faceDimQuery = await queryRunner.query(` - SELECT CARDINALITY("embedding") - FROM "asset_faces" - LIMIT 1`); - const clipDimQuery = await queryRunner.query(` - SELECT CARDINALITY("clipEmbedding") - FROM "smart_info" - LIMIT 1`); - - const faceDimSize = faceDimQuery[0] ?? 512; - const clipDimSize = clipDimQuery[0] ?? 512; - - await queryRunner.query(` - ALTER TABLE "asset_faces" - ALTER COLUMN "embedding" TYPE vector(${faceDimSize})`); - await queryRunner.query( - `ALTER TABLE "smart_info" - ALTER COLUMN "clipEmbedding" TYPE vector(${clipDimSize})`, - ); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "asset_faces" ALTER COLUMN "embedding" TYPE real array`); - await queryRunner.query(`ALTER TABLE "smart_info" ALTER COLUMN "clipEmbedding" TYPE real array`); - } -} diff --git a/server/src/infra/migrations/1696875736010-AddSmartInfoTextSearchColumn.ts b/server/src/infra/migrations/1696875736010-AddSmartInfoTextSearchColumn.ts deleted file mode 100644 index a31b5f20f..000000000 --- a/server/src/infra/migrations/1696875736010-AddSmartInfoTextSearchColumn.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class AddSmartInfoTextSearchColumn1696875736010 implements MigrationInterface { - name = 'AddSmartInfoTextSearchColumn1696875736010'; - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(` - CREATE OR REPLACE FUNCTION immutable_concat_ws(text, text[]) - RETURNS text - LANGUAGE internal IMMUTABLE PARALLEL SAFE AS - 'text_concat_ws'`); - await queryRunner.query(` - ALTER TABLE "smart_info" ADD "smartInfoTextSearchableColumn" tsvector - GENERATED ALWAYS AS ( - TO_TSVECTOR( - 'english', - immutable_concat_ws( - ' '::text, - COALESCE(tags, array[]::text[]) || COALESCE(objects, array[]::text[]) - ) - ) - ) - STORED NOT NULL`); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP FUNCTION IF EXISTS immutable_concat_ws(text, text[])`); - await queryRunner.query(`ALTER TABLE "smart_info" DROP COLUMN IF EXISTS "smartInfoTextSearchableColumn"`); - } -} diff --git a/server/src/infra/migrations/1696876192604-CreateSmartInfoTextSearchIndex.ts b/server/src/infra/migrations/1696876192604-CreateSmartInfoTextSearchIndex.ts index f030b5204..4727fbeb3 100644 --- a/server/src/infra/migrations/1696876192604-CreateSmartInfoTextSearchIndex.ts +++ b/server/src/infra/migrations/1696876192604-CreateSmartInfoTextSearchIndex.ts @@ -4,6 +4,25 @@ export class CreateSmartInfoTextSearchIndex1696876192604 implements MigrationInt name = 'CreateSmartInfoTextSearchIndex1696876192604'; public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE OR REPLACE FUNCTION immutable_concat_ws(text, text[]) + RETURNS text + LANGUAGE internal IMMUTABLE PARALLEL SAFE AS + 'text_concat_ws'`); + + await queryRunner.query(` + ALTER TABLE smart_info ADD "smartInfoTextSearchableColumn" tsvector + GENERATED ALWAYS AS ( + TO_TSVECTOR( + 'english', + immutable_concat_ws( + ' '::text, + COALESCE(tags, array[]::text[]) || COALESCE(objects, array[]::text[]) + ) + ) + ) + STORED NOT NULL`); + await queryRunner.query(` CREATE INDEX smart_info_text_searchable_idx ON smart_info @@ -11,6 +30,7 @@ export class CreateSmartInfoTextSearchIndex1696876192604 implements MigrationInt } public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP INDEX IF EXISTS smart_info_text_searchable_idx ON smart_info`); + await queryRunner.query(`DROP FUNCTION IF EXISTS immutable_concat_ws`); + await queryRunner.query(`ALTER TABLE smart_info DROP IF EXISTS "smartInfoTextSearchableColumn"`); } } diff --git a/server/src/infra/migrations/1699419684990-AddExifCityIndex.ts b/server/src/infra/migrations/1699419684990-AddExifCityIndex.ts deleted file mode 100644 index 19a5598a2..000000000 --- a/server/src/infra/migrations/1699419684990-AddExifCityIndex.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; - -export class AddExifCityIndex1699419684990 implements MigrationInterface { - name = 'AddExifCityIndex1699419684990' - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE INDEX IF NOT EXISTS exif_city ON exif (city);`); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP INDEX IF EXISTS exif_city;`); - } -} diff --git a/server/src/infra/migrations/1699419700539-AddSmartInfoTagsIndex.ts b/server/src/infra/migrations/1699419700539-AddSmartInfoTagsIndex.ts deleted file mode 100644 index 0c92595ce..000000000 --- a/server/src/infra/migrations/1699419700539-AddSmartInfoTagsIndex.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; - -export class AddSmartInfoTagsIndex1699419700539 implements MigrationInterface { - name = 'AddSmartInfoTagsIndex1699419700539' - - public async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`CREATE INDEX IF NOT EXISTS si_tags ON smart_info USING GIN (tags);`); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP INDEX IF EXISTS si_tags;`); - } -} diff --git a/server/src/infra/migrations/1699586761207-AddCLIPEmbeddingIndex.ts b/server/src/infra/migrations/1699586761207-AddCLIPEmbeddingIndex.ts deleted file mode 100644 index 337065c15..000000000 --- a/server/src/infra/migrations/1699586761207-AddCLIPEmbeddingIndex.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { MigrationInterface, QueryRunner } from 'typeorm'; - -export class AddCLIPEmbeddingIndex1699586761207 implements MigrationInterface { - name = 'AddCLIPEmbeddingIndex1699586761207'; - - public async up(queryRunner: QueryRunner): Promise { - console.log('Creating CLIP index. This may take a while...'); - await queryRunner.query(` - CREATE INDEX IF NOT EXISTS clip_index - ON smart_info - USING hnsw ("clipEmbedding" vector_ip_ops) - WITH (m = 16, ef_construction = 128)`); - await queryRunner.query(`SET hnsw.ef_search = 250`); - } - - public async down(queryRunner: QueryRunner): Promise { - await queryRunner.query(`DROP INDEX IF EXISTS clip_index`); - } -} diff --git a/server/src/infra/migrations/1699746198141-UsePgVector.ts b/server/src/infra/migrations/1699746198141-UsePgVector.ts new file mode 100644 index 000000000..362e547e2 --- /dev/null +++ b/server/src/infra/migrations/1699746198141-UsePgVector.ts @@ -0,0 +1,47 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class UsePgVector1699746198141 implements MigrationInterface { + name = 'UsePgVector1699746198141'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query('DROP EXTENSION IF EXISTS vectors'); + await queryRunner.query('CREATE EXTENSION vectors'); + + const faceDimQuery = await queryRunner.query(` + SELECT CARDINALITY(embedding) as dimsize + FROM asset_faces + LIMIT 1`); + const clipDimQuery = await queryRunner.query(` + SELECT CARDINALITY("clipEmbedding") as dimsize + FROM smart_info + LIMIT 1`); + + const faceDimSize = faceDimQuery?.[0]?.['dimsize'] ?? 512; + const clipDimSize = clipDimQuery?.[0]?.['dimsize'] ?? 512; + + await queryRunner.query(`ALTER TABLE asset_faces ALTER COLUMN embedding TYPE vector(${faceDimSize})`); + await queryRunner.query(`CREATE TABLE smart_search ( + "assetId" uuid PRIMARY KEY NOT NULL REFERENCES assets(id) ON DELETE CASCADE, + embedding vector(${clipDimSize}) NOT NULL )`); + await queryRunner.query(` + INSERT INTO smart_search("assetId", embedding) + SELECT si."assetId", si."clipEmbedding" + FROM smart_info si + WHERE "clipEmbedding" IS NOT NULL + `); + await queryRunner.query(`ALTER TABLE smart_info DROP COLUMN "clipEmbedding"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE asset_faces ALTER COLUMN embedding TYPE real array`); + await queryRunner.query(`ALTER TABLE smart_info ADD COLUMN IF NOT EXISTS "clipEmbedding" TYPE real array`); + await queryRunner.query(` + INSERT INTO smart_info + ("assetId", "clipEmbedding") + SELECT s."assetId", s.embedding + FROM smart_search s + ON CONFLICT (s."assetId") DO UPDATE SET "clipEmbedding" = s.embedding + `); + await queryRunner.query(`DROP TABLE IF EXISTS smart_search`); + } +} diff --git a/server/src/infra/migrations/1699746301742-AddCLIPEmbeddingIndex.ts b/server/src/infra/migrations/1699746301742-AddCLIPEmbeddingIndex.ts new file mode 100644 index 000000000..aa0f4bb40 --- /dev/null +++ b/server/src/infra/migrations/1699746301742-AddCLIPEmbeddingIndex.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddCLIPEmbeddingIndex1699746301742 implements MigrationInterface { + name = 'AddCLIPEmbeddingIndex1699746301742'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS clip_index ON smart_search + USING vectors (embedding cosine_ops) WITH (options = $$ + capacity = 2097152 + [indexing.hnsw] + m = 16 + ef_construction = 300 + $$);`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX IF EXISTS clip_index`); + } +} diff --git a/server/src/infra/migrations/1699746316571-AddExifCityIndex.ts b/server/src/infra/migrations/1699746316571-AddExifCityIndex.ts new file mode 100644 index 000000000..3f1ce98de --- /dev/null +++ b/server/src/infra/migrations/1699746316571-AddExifCityIndex.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddExifCityIndex1699746316571 implements MigrationInterface { + name = 'AddExifCityIndex1699746316571'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX IF NOT EXISTS exif_city ON exif (city);`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX IF EXISTS exif_city;`); + } +} diff --git a/server/src/infra/migrations/1699746328479-AddSmartInfoTagsIndex.ts b/server/src/infra/migrations/1699746328479-AddSmartInfoTagsIndex.ts new file mode 100644 index 000000000..13ba0da3d --- /dev/null +++ b/server/src/infra/migrations/1699746328479-AddSmartInfoTagsIndex.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddSmartInfoTagsIndex1699746328479 implements MigrationInterface { + name = 'AddSmartInfoTagsIndex1699746328479'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX IF NOT EXISTS si_tags ON smart_info USING GIN (tags);`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX IF EXISTS si_tags;`); + } +} diff --git a/server/src/infra/migrations/1699746444644-AddFaceEmbeddingIndex.ts b/server/src/infra/migrations/1699746444644-AddFaceEmbeddingIndex.ts new file mode 100644 index 000000000..60af3cfb9 --- /dev/null +++ b/server/src/infra/migrations/1699746444644-AddFaceEmbeddingIndex.ts @@ -0,0 +1,20 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddFaceEmbeddingIndex1699746444644 implements MigrationInterface { + name = 'AddFaceEmbeddingIndex1699746444644'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE INDEX IF NOT EXISTS face_index ON asset_faces + USING vectors (embedding cosine_ops) WITH (options = $$ + capacity = 2097152 + [indexing.hnsw] + m = 16 + ef_construction = 300 + $$);`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX IF EXISTS face_index`); + } +} diff --git a/server/src/infra/repositories/asset.repository.ts b/server/src/infra/repositories/asset.repository.ts index 6a04c103c..5337c94dc 100644 --- a/server/src/infra/repositories/asset.repository.ts +++ b/server/src/infra/repositories/asset.repository.ts @@ -477,13 +477,13 @@ export class AssetRepository implements IAssetRepository { case WithoutProperty.CLIP_ENCODING: relations = { - smartInfo: true, + smartSearch: true, }; where = { isVisible: true, resizePath: Not(IsNull()), - smartInfo: { - clipEmbedding: IsNull(), + smartSearch: { + embedding: IsNull(), }, }; break; diff --git a/server/src/infra/repositories/smart-info.repository.ts b/server/src/infra/repositories/smart-info.repository.ts index 050ba5478..8e5abe137 100644 --- a/server/src/infra/repositories/smart-info.repository.ts +++ b/server/src/infra/repositories/smart-info.repository.ts @@ -1,9 +1,9 @@ -import { EmbeddingSearch, ISmartInfoRepository } from '@app/domain'; +import { Embedding, EmbeddingSearch, ISmartInfoRepository } from '@app/domain'; import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import AsyncLock from 'async-lock'; import { Repository } from 'typeorm'; -import { AssetEntity, SmartInfoEntity } from '../entities'; +import { AssetEntity, SmartInfoEntity, SmartSearchEntity } from '../entities'; import { asVector } from '../infra.utils'; @Injectable() @@ -12,70 +12,72 @@ export class SmartInfoRepository implements ISmartInfoRepository { private lock: AsyncLock; private curDimSize: number | undefined; - constructor(@InjectRepository(SmartInfoEntity) private repository: Repository) { + constructor( + @InjectRepository(SmartInfoEntity) private repository: Repository, + @InjectRepository(AssetEntity) private assetRepository: Repository, + @InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository) { this.lock = new AsyncLock(); } async searchByEmbedding({ ownerId, embedding, numResults }: EmbeddingSearch): Promise { - const results = await this.repository - .createQueryBuilder('smartInfo') - .useTransaction(true) - .leftJoinAndSelect('smartInfo.asset', 'asset') - .where('asset.ownerId = :ownerId', { ownerId }) - .orderBy(`smartInfo.clipEmbedding <=> :embedding`) - .setParameters({ embedding: asVector(embedding) }) - .limit(numResults) - .getMany(); + let results: AssetEntity[] = await this.assetRepository.createQueryBuilder('a') + .innerJoin('a.smartSearch', 's') + .where('a.ownerId = :ownerId') + .leftJoinAndSelect('a.exifInfo', 'e') + .orderBy('s.embedding <=> :embedding') + .setParameters({ embedding: asVector(embedding), ownerId }) + .limit(numResults) + .getMany(); - return results.map((result) => result.asset).filter((asset): asset is AssetEntity => !!asset); + return results; } - async upsert(info: Partial): Promise { - const { clipEmbedding, ...withoutEmbedding } = info; - await this.repository.upsert(withoutEmbedding, { conflictPaths: ['assetId'] }); - if (!clipEmbedding || !info.assetId) return; + async upsert(smartInfo: Partial, embedding?: Embedding): Promise { + await this.repository.upsert(smartInfo, { conflictPaths: ['assetId'] }); + if (!smartInfo.assetId || !embedding) return; try { - await this.updateEmbedding(clipEmbedding, info.assetId); + await this.upsertEmbedding(smartInfo.assetId, embedding); } catch (e) { - await this.updateDimSize(clipEmbedding.length); - await this.updateEmbedding(clipEmbedding, info.assetId); + await this.updateDimSize(embedding.length); + await this.upsertEmbedding(smartInfo.assetId, embedding); } } - private async updateEmbedding(embedding: number[], assetId: string): Promise { - await this.repository.manager.query(`UPDATE "smart_info" SET "clipEmbedding" = $1 WHERE "assetId" = $2`, [ - asVector(embedding), - assetId, - ]); + private async upsertEmbedding(assetId: string, embedding: number[]): Promise { + await this.smartSearchRepository.manager.query( + `INSERT INTO smart_search ($1, $2) ON CONFLICT ("assetId") SET embedding = $2`, + [assetId, asVector(embedding)], + ); } /* * note: never use this with user input * this does not parameterize the query because it is not possible to parameterize the column type - */ + */ private async updateDimSize(dimSize: number): Promise { await this.lock.acquire('updateDimSizeLock', async () => { if (this.curDimSize === dimSize) return; this.logger.log(`Updating CLIP dimension size to ${dimSize}`); - await this.repository.manager.query(` + await this.smartSearchRepository.manager.query(` BEGIN; - ALTER TABLE smart_info - DROP COLUMN "clipEmbedding", - ADD COLUMN "clipEmbedding" vector(${dimSize}); + ALTER TABLE smart_search + DROP COLUMN embedding, + ADD COLUMN embedding vector(${dimSize}); - CREATE INDEX IF NOT EXISTS clip_index - ON smart_info - USING hnsw ("clipEmbedding" vector_ip_ops) - WITH (m = 16, ef_construction = 128); + CREATE INDEX clip_index ON smart_search + USING vectors (embedding dot_ops) WITH (options = $$ + capacity = 2097152 + [indexing.hnsw] + m = 16 + ef_construction = 300 + $$); - SET hnsw.ef_search = 250; COMMIT; - ` - ); + `); this.curDimSize = dimSize; this.logger.log(`Successfully updated CLIP dimension size to ${dimSize}`); diff --git a/web/package-lock.json b/web/package-lock.json index 5f5a3982f..9391f1f6c 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -29,6 +29,7 @@ "devDependencies": { "@babel/preset-env": "^7.20.2", "@babel/preset-typescript": "^7.22.5", + "@esbuild/linux-x64": "^0.19.5", "@faker-js/faker": "^8.0.0", "@floating-ui/dom": "^1.5.1", "@sveltejs/adapter-static": "^2.0.3", @@ -1882,6 +1883,54 @@ "gl-matrix": "^3.4.3" } }, + "node_modules/@esbuild/android-arm": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", + "integrity": "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz", + "integrity": "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.20.tgz", + "integrity": "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@esbuild/darwin-arm64": { "version": "0.18.20", "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz", @@ -1898,6 +1947,293 @@ "node": ">=12" } }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz", + "integrity": "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz", + "integrity": "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz", + "integrity": "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz", + "integrity": "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz", + "integrity": "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz", + "integrity": "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz", + "integrity": "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz", + "integrity": "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz", + "integrity": "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz", + "integrity": "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz", + "integrity": "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.19.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.5.tgz", + "integrity": "sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A==", + "cpu": [ + "x64" + ], + "dev": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz", + "integrity": "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz", + "integrity": "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz", + "integrity": "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz", + "integrity": "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz", + "integrity": "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz", + "integrity": "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -5413,6 +5749,22 @@ "@esbuild/win32-x64": "0.18.20" } }, + "node_modules/esbuild/node_modules/@esbuild/linux-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz", + "integrity": "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", diff --git a/web/package.json b/web/package.json index db7b2e8cf..6ceb1ea8e 100644 --- a/web/package.json +++ b/web/package.json @@ -22,6 +22,7 @@ "devDependencies": { "@babel/preset-env": "^7.20.2", "@babel/preset-typescript": "^7.22.5", + "@esbuild/linux-x64": "^0.19.5", "@faker-js/faker": "^8.0.0", "@floating-ui/dom": "^1.5.1", "@sveltejs/adapter-static": "^2.0.3",