review suggestions, vector migrations on bootstrap

This commit is contained in:
mertalev 2023-11-22 21:18:39 -05:00
parent 1a25f3fef7
commit 4ec6d107b1
No known key found for this signature in database
GPG key ID: 9181CD92C0A1C5E3
7 changed files with 19 additions and 21 deletions

View file

@ -1,7 +1,7 @@
import { SearchExploreItem } from '@app/domain';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities'; import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities';
import { FindOptionsRelations } from 'typeorm'; import { FindOptionsRelations } from 'typeorm';
import { Paginated, PaginationOptions } from '../domain.util'; import { Paginated, PaginationOptions } from '../domain.util';
import { SearchExploreItem } from '../repositories';
export type AssetStats = Record<AssetType, number>; export type AssetStats = Record<AssetType, number>;

View file

@ -1,5 +1,5 @@
import { Embedding, EmbeddingSearch } from '@app/domain';
import { AssetEntity, SmartInfoEntity } from '@app/infra/entities'; import { AssetEntity, SmartInfoEntity } from '@app/infra/entities';
import { Embedding, EmbeddingSearch } from '../repositories';
export const ISmartInfoRepository = 'ISmartInfoRepository'; export const ISmartInfoRepository = 'ISmartInfoRepository';

View file

@ -18,7 +18,6 @@ import { SearchResponseDto } from './response-dto';
@Injectable() @Injectable()
export class SearchService { export class SearchService {
private timer: NodeJS.Timeout | null = null;
private logger = new Logger(SearchService.name); private logger = new Logger(SearchService.name);
private configCore: SystemConfigCore; private configCore: SystemConfigCore;

View file

@ -1,5 +1,5 @@
import { envName, getLogLevels, isDev, serverVersion } from '@app/domain'; import { envName, getLogLevels, isDev, serverVersion } from '@app/domain';
import { RedisIoAdapter } from '@app/infra'; import { RedisIoAdapter, runVectorMigrations } from '@app/infra';
import { Logger } from '@nestjs/common'; import { Logger } from '@nestjs/common';
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { NestExpressApplication } from '@nestjs/platform-express'; import { NestExpressApplication } from '@nestjs/platform-express';
@ -29,6 +29,8 @@ export async function bootstrap() {
app.useStaticAssets('www'); app.useStaticAssets('www');
app.use(indexFallback(excludePaths)); app.use(indexFallback(excludePaths));
await runVectorMigrations();
const server = await app.listen(port); const server = await app.listen(port);
server.requestTimeout = 30 * 60 * 1000; server.requestTimeout = 30 * 60 * 1000;

View file

@ -29,8 +29,13 @@ export const databaseConfigVector: PostgresConnectionOptions = {
migrations: [__dirname + '/migrations/vector/*.{js,ts}'], migrations: [__dirname + '/migrations/vector/*.{js,ts}'],
}; };
export async function initDataSource(): Promise<DataSource> { // this export is used by TypeORM commands in package.json#scripts
const dataSource = await new DataSource(databaseConfig).initialize(); export let dataSource = new DataSource(databaseConfig);
export async function runVectorMigrations(): Promise<void> {
if (!dataSource.isInitialized) {
dataSource = await dataSource.initialize();
}
const hasVectorExtension = (await dataSource.query( const hasVectorExtension = (await dataSource.query(
`SELECT * FROM pg_available_extensions WHERE name = 'vectors'`, `SELECT * FROM pg_available_extensions WHERE name = 'vectors'`,
@ -44,12 +49,4 @@ export async function initDataSource(): Promise<DataSource> {
await dataSourceVector.destroy(); await dataSourceVector.destroy();
} }
return dataSource;
} }
// this export is used by TypeORM commands in package.json#scripts
export let dataSource: DataSource;
(async () => {
dataSource = await initDataSource();
})();

View file

@ -20,9 +20,9 @@ import { ExifEntity } from './exif.entity';
import { LibraryEntity } from './library.entity'; import { LibraryEntity } from './library.entity';
import { SharedLinkEntity } from './shared-link.entity'; import { SharedLinkEntity } from './shared-link.entity';
import { SmartInfoEntity } from './smart-info.entity'; import { SmartInfoEntity } from './smart-info.entity';
import { SmartSearchEntity } from './smart-search.entity';
import { TagEntity } from './tag.entity'; import { TagEntity } from './tag.entity';
import { UserEntity } from './user.entity'; import { UserEntity } from './user.entity';
import { SmartSearchEntity } from '.';
export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum'; export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum';

View file

@ -89,12 +89,12 @@ export class SmartInfoRepository implements ISmartInfoRepository {
embedding vector(${dimSize}) NOT NULL )`); embedding vector(${dimSize}) NOT NULL )`);
await manager.query(` await manager.query(`
CREATE INDEX clip_index ON smart_search CREATE INDEX clip_index ON smart_search
USING vectors (embedding cosine_ops) WITH (options = $$ USING vectors (embedding cosine_ops) WITH (options = $$
[indexing.hnsw] [indexing.hnsw]
m = 16 m = 16
ef_construction = 300 ef_construction = 300
$$)`); $$)`);
}); });
this.curDimSize = dimSize; this.curDimSize = dimSize;