beenull/immich
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

clip search wip

Browse source
This commit is contained in:
mertalev 2023-09-11 01:54:22 -04:00
parent b26b4042cf
commit 1ea5dcc469
No known key found for this signature in database
GPG key ID: 9181CD92C0A1C5E3
22 changed files with 1429 additions and 1760 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
machine-learning/app/main.py
View file

@ -2,7 +2,7 @@ import asyncio
from functools import partial
import threading
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Callable
from typing import Any, Callable, Type
from zipfile import BadZipFile
import faiss
@ -28,7 +28,23 @@ app = FastAPI()
vector_stores: dict[str, faiss.IndexIDMap2] = {}
def validate_embeddings(embeddings: list[float] | np.ndarray[int, np.dtype[Any]]) -> np.ndarray[int, np.dtype[Any]]:
class VectorStore:
def __init__(self, dims: int, index_cls: Type[faiss.Index] = faiss.IndexHNSWFlat, **kwargs: Any) -> None:
self.index = index_cls(dims, **kwargs)
self.id_to_key: dict[int, Any] = {}
def search(self, embeddings: np.ndarray[int, np.dtype[Any]], k: int) -> list[Any]:
ids = self.index.assign(embeddings, k) # type: ignore
return [self.id_to_key[idx] for row in ids.tolist() for idx in row if not idx == -1]
def add_with_ids(self, embeddings: np.ndarray[int, np.dtype[Any]], embedding_ids: list[Any]) -> None:
self.id_to_key |= {
id: key for id, key in zip(embedding_ids, range(self.index.ntotal, self.index.ntotal + len(embedding_ids)))
}
self.index.add(embeddings) # type: ignore
def validate_embeddings(embeddings: list[float]) -> Any:
embeddings = np.array(embeddings)
if len(embeddings.shape) == 1:
embeddings = np.expand_dims(embeddings, 0)
@ -91,14 +107,19 @@ async def pipeline(
except orjson.JSONDecodeError:
raise HTTPException(400, f"Invalid options JSON: {options}")
outputs = await run(_predict, model_name, model_type, inputs, **kwargs)
outputs = await _predict(model_name, model_type, inputs, **kwargs)
if index_name is not None:
if k is not None:
if k < 1:
raise HTTPException(400, f"k must be a positive integer; got {k}")
outputs = await run(_search, index_name, outputs, k)
if index_name not in vector_stores:
raise HTTPException(404, f"Index '{index_name}' not found")
outputs = await run(vector_stores[index_name].search, outputs, k)
if embedding_id is not None:
await run(_add, index_name, [embedding_id], outputs)
if index_name not in vector_stores:
await create(index_name, [embedding_id], outputs)
else:
await run(vector_stores[index_name].add, [embedding_id], outputs)
return ORJSONResponse(outputs)
@ -121,17 +142,15 @@ async def predict(
except orjson.JSONDecodeError:
raise HTTPException(400, f"Invalid options JSON: {options}")
outputs = await run(_predict, model_name, model_type, inputs, **kwargs)
outputs = await _predict(model_name, model_type, inputs, **kwargs)
return ORJSONResponse(outputs)
@app.post("/index/{index_name}/search", response_class=ORJSONResponse)
async def search(
index_name: str, embeddings: np.ndarray[int, np.dtype[np.float32]] = Depends(validate_embeddings), k: int = 10
) -> ORJSONResponse:
async def search(index_name: str, embeddings: Any = Depends(validate_embeddings), k: int = 10) -> ORJSONResponse:
if index_name not in vector_stores or vector_stores[index_name].d != embeddings.shape[1]:
raise HTTPException(404, f"Index '{index_name}' not found")
outputs: np.ndarray[int, np.dtype[Any]] = await run(_search, index_name, embeddings, k)
outputs: np.ndarray[int, np.dtype[Any]] = await run(vector_stores[index_name].search, embeddings, k)
return ORJSONResponse(outputs)
@ -139,19 +158,19 @@ async def search(
async def add(
index_name: str,
embedding_ids: list[str],
embeddings: np.ndarray[int, np.dtype[np.float32]] = Depends(validate_embeddings),
embeddings: Any = Depends(validate_embeddings),
) -> None:
if index_name not in vector_stores or vector_stores[index_name].d != embeddings.shape[1]:
await create(index_name, embedding_ids, embeddings)
else:
await run(_add, index_name, embedding_ids, embeddings)
await run(vector_stores[index_name].add_with_ids, embeddings, embedding_ids)
@app.post("/index/{index_name}/create")
async def create(
index_name: str,
embedding_ids: list[str],
embeddings: np.ndarray[int, np.dtype[np.float32]] = Depends(validate_embeddings),
embeddings: Any = Depends(validate_embeddings),
) -> None:
if embeddings.shape[0] != len(embedding_ids):
raise HTTPException(400, "Number of embedding IDs must match number of embeddings")
@ -169,7 +188,7 @@ async def run(func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
return await asyncio.get_running_loop().run_in_executor(app.state.thread_pool, func, *args)
async def _load(model: InferenceModel) -> InferenceModel:
def _load(model: InferenceModel) -> InferenceModel:
if model.loaded:
return model
@ -189,31 +208,22 @@ async def _load(model: InferenceModel) -> InferenceModel:
return model
async def _add(index_name: str, embedding_ids: list[str], embeddings: np.ndarray[int, np.dtype[np.float32]]) -> None:
return await vector_stores[index_name].add_with_ids(embeddings, embedding_ids) # type: ignore
async def _search(
index_name: str, embeddings: np.ndarray[int, np.dtype[np.float32]], k: int
) -> np.ndarray[int, np.dtype[Any]]:
return await vector_stores[index_name].assign(embeddings, k) # type: ignore
async def _predict(
model_name: str, model_type: ModelType, inputs: Any, **options: Any
) -> np.ndarray[int, np.dtype[np.float32]]:
model = await _load(await app.state.model_cache.get(model_name, model_type, **options))
model = await app.state.model_cache.get(model_name, model_type, **options)
if not model.loaded:
await run(_load, model)
model.configure(**options)
return await run(model.predict, inputs)
async def _create(
def _create(
embedding_ids: list[str],
embeddings: np.ndarray[int, np.dtype[np.float32]],
) -> faiss.IndexIDMap2:
hnsw_index = faiss.IndexHNSWFlat(embeddings.shape[1])
mapped_index = faiss.IndexIDMap2(hnsw_index)
) -> VectorStore:
index = VectorStore(embeddings.shape[1])
with app.state.index_lock:
mapped_index.add_with_ids(embeddings, embedding_ids) # type: ignore
return mapped_index
index.add_with_ids(embeddings, embedding_ids) # type: ignore
return index

6
server/src/domain/album/album.service.ts
View file

@ -104,7 +104,7 @@ export class AlbumService {
albumThumbnailAssetId: dto.assetIds?.[0] || null,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [album.id] } });
// await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [album.id] } });
return mapAlbumWithAssets(album);
}
@ -127,7 +127,7 @@ export class AlbumService {
albumThumbnailAssetId: dto.albumThumbnailAssetId,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [updatedAlbum.id] } });
// await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [updatedAlbum.id] } });
return mapAlbumWithoutAssets(updatedAlbum);
}
@ -138,7 +138,7 @@ export class AlbumService {
const album = await this.findOrFail(id, { withAssets: false });
await this.albumRepository.delete(album);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ALBUM, data: { ids: [id] } });
// await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ALBUM, data: { ids: [id] } });
}
async addAssets(authUser: AuthUserDto, id: string, dto: BulkIdsDto): Promise<BulkIdResponseDto[]> {

2
server/src/domain/asset/asset.service.ts
View file

@ -289,7 +289,7 @@ export class AssetService {
}
const asset = await this.assetRepository.save({ id, ...rest });
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [id] } });
// await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [id] } });
return mapAsset(asset);
}

8
server/src/domain/domain.module.ts
View file

@ -52,9 +52,7 @@ const providers: Provider[] = [
@Global()
@Module({})
export class DomainModule implements OnApplicationShutdown {
constructor(private searchService: SearchService) {}
export class DomainModule {
static register(options: Pick<ModuleMetadata, 'imports'>): DynamicModule {
return {
module: DomainModule,
@ -63,8 +61,4 @@ export class DomainModule implements OnApplicationShutdown {
exports: [...providers],
};
}
onApplicationShutdown() {
this.searchService.teardown();
}
}

652
server/src/domain/facial-recognition/facial-recognition.service.spec.ts
View file

@ -1,367 +1,367 @@
import { Colorspace, SystemConfigKey } from '@app/infra/entities';
import {
assetStub,
faceStub,
newAssetRepositoryMock,
newFaceRepositoryMock,
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newMediaRepositoryMock,
newPersonRepositoryMock,
newSearchRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
personStub,
} from '@test';
import { IAssetRepository, WithoutProperty } from '../asset';
import { IJobRepository, JobName } from '../job';
import { IMediaRepository } from '../media';
import { IPersonRepository } from '../person';
import { ISearchRepository } from '../search';
import { IMachineLearningRepository } from '../smart-info';
import { IStorageRepository } from '../storage';
import { ISystemConfigRepository } from '../system-config';
import { IFaceRepository } from './face.repository';
import { FacialRecognitionService } from './facial-recognition.services';
// import { Colorspace, SystemConfigKey } from '@app/infra/entities';
// import {
// assetStub,
// faceStub,
// newAssetRepositoryMock,
// newFaceRepositoryMock,
// newJobRepositoryMock,
// newMachineLearningRepositoryMock,
// newMediaRepositoryMock,
// newPersonRepositoryMock,
// newSearchRepositoryMock,
// newStorageRepositoryMock,
// newSystemConfigRepositoryMock,
// personStub,
// } from '@test';
// import { IAssetRepository, WithoutProperty } from '../asset';
// import { IJobRepository, JobName } from '../job';
// import { IMediaRepository } from '../media';
// import { IPersonRepository } from '../person';
// import { ISearchRepository } from '../search';
// import { IMachineLearningRepository } from '../smart-info';
// import { IStorageRepository } from '../storage';
// import { ISystemConfigRepository } from '../system-config';
// import { IFaceRepository } from './face.repository';
// import { FacialRecognitionService } from './facial-recognition.services';
const croppedFace = Buffer.from('Cropped Face');
// const croppedFace = Buffer.from('Cropped Face');
const face = {
start: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 5,
y1: 5,
x2: 505,
y2: 505,
},
imageHeight: 1000,
imageWidth: 1000,
},
middle: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 100,
y1: 100,
x2: 200,
y2: 200,
},
imageHeight: 500,
imageWidth: 400,
embedding: [1, 2, 3, 4],
score: 0.2,
},
end: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 300,
y1: 300,
x2: 495,
y2: 495,
},
imageHeight: 500,
imageWidth: 500,
},
};
// const face = {
// start: {
// assetId: 'asset-1',
// personId: 'person-1',
// boundingBox: {
// x1: 5,
// y1: 5,
// x2: 505,
// y2: 505,
// },
// imageHeight: 1000,
// imageWidth: 1000,
// },
// middle: {
// assetId: 'asset-1',
// personId: 'person-1',
// boundingBox: {
// x1: 100,
// y1: 100,
// x2: 200,
// y2: 200,
// },
// imageHeight: 500,
// imageWidth: 400,
// embedding: [1, 2, 3, 4],
// score: 0.2,
// },
// end: {
// assetId: 'asset-1',
// personId: 'person-1',
// boundingBox: {
// x1: 300,
// y1: 300,
// x2: 495,
// y2: 495,
// },
// imageHeight: 500,
// imageWidth: 500,
// },
// };
const faceSearch = {
noMatch: {
total: 0,
count: 0,
page: 1,
items: [],
distances: [],
facets: [],
},
oneMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.1],
facets: [],
},
oneRemoteMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.8],
facets: [],
},
};
// const faceSearch = {
// noMatch: {
// total: 0,
// count: 0,
// page: 1,
// items: [],
// distances: [],
// facets: [],
// },
// oneMatch: {
// total: 1,
// count: 1,
// page: 1,
// items: [faceStub.face1],
// distances: [0.1],
// facets: [],
// },
// oneRemoteMatch: {
// total: 1,
// count: 1,
// page: 1,
// items: [faceStub.face1],
// distances: [0.8],
// facets: [],
// },
// };
describe(FacialRecognitionService.name, () => {
let sut: FacialRecognitionService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let faceMock: jest.Mocked<IFaceRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let machineLearningMock: jest.Mocked<IMachineLearningRepository>;
let mediaMock: jest.Mocked<IMediaRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
// describe(FacialRecognitionService.name, () => {
// let sut: FacialRecognitionService;
// let assetMock: jest.Mocked<IAssetRepository>;
// let configMock: jest.Mocked<ISystemConfigRepository>;
// let faceMock: jest.Mocked<IFaceRepository>;
// let jobMock: jest.Mocked<IJobRepository>;
// let machineLearningMock: jest.Mocked<IMachineLearningRepository>;
// let mediaMock: jest.Mocked<IMediaRepository>;
// let personMock: jest.Mocked<IPersonRepository>;
// let searchMock: jest.Mocked<ISearchRepository>;
// let storageMock: jest.Mocked<IStorageRepository>;
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
faceMock = newFaceRepositoryMock();
jobMock = newJobRepositoryMock();
machineLearningMock = newMachineLearningRepositoryMock();
mediaMock = newMediaRepositoryMock();
personMock = newPersonRepositoryMock();
searchMock = newSearchRepositoryMock();
storageMock = newStorageRepositoryMock();
// beforeEach(async () => {
// assetMock = newAssetRepositoryMock();
// configMock = newSystemConfigRepositoryMock();
// faceMock = newFaceRepositoryMock();
// jobMock = newJobRepositoryMock();
// machineLearningMock = newMachineLearningRepositoryMock();
// mediaMock = newMediaRepositoryMock();
// personMock = newPersonRepositoryMock();
// searchMock = newSearchRepositoryMock();
// storageMock = newStorageRepositoryMock();
mediaMock.crop.mockResolvedValue(croppedFace);
// mediaMock.crop.mockResolvedValue(croppedFace);
sut = new FacialRecognitionService(
assetMock,
configMock,
faceMock,
jobMock,
machineLearningMock,
mediaMock,
personMock,
searchMock,
storageMock,
);
});
// sut = new FacialRecognitionService(
// assetMock,
// configMock,
// faceMock,
// jobMock,
// machineLearningMock,
// mediaMock,
// personMock,
// searchMock,
// storageMock,
// );
// });
it('should be defined', () => {
expect(sut).toBeDefined();
});
// it('should be defined', () => {
// expect(sut).toBeDefined();
// });
describe('handleQueueRecognizeFaces', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
// describe('handleQueueRecognizeFaces', () => {
// it('should return if machine learning is disabled', async () => {
// configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true);
expect(jobMock.queue).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled();
});
// await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(true);
// expect(jobMock.queue).not.toHaveBeenCalled();
// expect(configMock.load).toHaveBeenCalled();
// });
it('should queue missing assets', async () => {
assetMock.getWithout.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
await sut.handleQueueRecognizeFaces({});
// it('should queue missing assets', async () => {
// assetMock.getWithout.mockResolvedValue({
// items: [assetStub.image],
// hasNextPage: false,
// });
// await sut.handleQueueRecognizeFaces({});
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.FACES);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.RECOGNIZE_FACES,
data: { id: assetStub.image.id },
});
});
// expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.FACES);
// expect(jobMock.queue).toHaveBeenCalledWith({
// name: JobName.RECOGNIZE_FACES,
// data: { id: assetStub.image.id },
// });
// });
it('should queue all assets', async () => {
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
personMock.deleteAll.mockResolvedValue(5);
searchMock.deleteAllFaces.mockResolvedValue(100);
// it('should queue all assets', async () => {
// assetMock.getAll.mockResolvedValue({
// items: [assetStub.image],
// hasNextPage: false,
// });
// personMock.deleteAll.mockResolvedValue(5);
// searchMock.deleteAllFaces.mockResolvedValue(100);
await sut.handleQueueRecognizeFaces({ force: true });
// await sut.handleQueueRecognizeFaces({ force: true });
expect(assetMock.getAll).toHaveBeenCalled();
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.RECOGNIZE_FACES,
data: { id: assetStub.image.id },
});
});
});
// expect(assetMock.getAll).toHaveBeenCalled();
// expect(jobMock.queue).toHaveBeenCalledWith({
// name: JobName.RECOGNIZE_FACES,
// data: { id: assetStub.image.id },
// });
// });
// });
describe('handleRecognizeFaces', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
// describe('handleRecognizeFaces', () => {
// it('should return if machine learning is disabled', async () => {
// configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await expect(sut.handleRecognizeFaces({ id: 'foo' })).resolves.toBe(true);
expect(assetMock.getByIds).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled();
});
// await expect(sut.handleRecognizeFaces({ id: 'foo' })).resolves.toBe(true);
// expect(assetMock.getByIds).not.toHaveBeenCalled();
// expect(configMock.load).toHaveBeenCalled();
// });
it('should skip when no resize path', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]);
await sut.handleRecognizeFaces({ id: assetStub.noResizePath.id });
expect(machineLearningMock.detectFaces).not.toHaveBeenCalled();
});
// it('should skip when no resize path', async () => {
// assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]);
// await sut.handleRecognizeFaces({ id: assetStub.noResizePath.id });
// expect(machineLearningMock.detectFaces).not.toHaveBeenCalled();
// });
it('should handle no results', async () => {
machineLearningMock.detectFaces.mockResolvedValue([]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
expect(machineLearningMock.detectFaces).toHaveBeenCalledWith(
'http://immich-machine-learning:3003',
{
imagePath: assetStub.image.resizePath,
},
{
enabled: true,
maxDistance: 0.6,
minScore: 0.7,
modelName: 'buffalo_l',
},
);
expect(faceMock.create).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
// it('should handle no results', async () => {
// machineLearningMock.detectFaces.mockResolvedValue([]);
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
// await sut.handleRecognizeFaces({ id: assetStub.image.id });
// expect(machineLearningMock.detectFaces).toHaveBeenCalledWith(
// 'http://immich-machine-learning:3003',
// {
// imagePath: assetStub.image.resizePath,
// },
// {
// enabled: true,
// maxDistance: 0.6,
// minScore: 0.7,
// modelName: 'buffalo_l',
// },
// );
// expect(faceMock.create).not.toHaveBeenCalled();
// expect(jobMock.queue).not.toHaveBeenCalled();
// });
it('should match existing people', async () => {
machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
// it('should match existing people', async () => {
// machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
// searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch);
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
// await sut.handleRecognizeFaces({ id: assetStub.image.id });
expect(faceMock.create).toHaveBeenCalledWith({
personId: 'person-1',
assetId: 'asset-id',
embedding: [1, 2, 3, 4],
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
imageHeight: 500,
imageWidth: 400,
});
});
// expect(faceMock.create).toHaveBeenCalledWith({
// personId: 'person-1',
// assetId: 'asset-id',
// embedding: [1, 2, 3, 4],
// boundingBoxX1: 100,
// boundingBoxY1: 100,
// boundingBoxX2: 200,
// boundingBoxY2: 200,
// imageHeight: 500,
// imageWidth: 400,
// });
// });
it('should create a new person', async () => {
machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch);
personMock.create.mockResolvedValue(personStub.noName);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
// it('should create a new person', async () => {
// machineLearningMock.detectFaces.mockResolvedValue([face.middle]);
// searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch);
// personMock.create.mockResolvedValue(personStub.noName);
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
// await sut.handleRecognizeFaces({ id: assetStub.image.id });
expect(personMock.create).toHaveBeenCalledWith({ ownerId: assetStub.image.ownerId });
expect(faceMock.create).toHaveBeenCalledWith({
personId: 'person-1',
assetId: 'asset-id',
embedding: [1, 2, 3, 4],
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
imageHeight: 500,
imageWidth: 400,
});
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.GENERATE_FACE_THUMBNAIL,
data: {
assetId: 'asset-1',
personId: 'person-1',
boundingBox: {
x1: 100,
y1: 100,
x2: 200,
y2: 200,
},
imageHeight: 500,
imageWidth: 400,
score: 0.2,
},
},
],
[{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
]);
});
});
// expect(personMock.create).toHaveBeenCalledWith({ ownerId: assetStub.image.ownerId });
// expect(faceMock.create).toHaveBeenCalledWith({
// personId: 'person-1',
// assetId: 'asset-id',
// embedding: [1, 2, 3, 4],
// boundingBoxX1: 100,
// boundingBoxY1: 100,
// boundingBoxX2: 200,
// boundingBoxY2: 200,
// imageHeight: 500,
// imageWidth: 400,
// });
// expect(jobMock.queue.mock.calls).toEqual([
// [
// {
// name: JobName.GENERATE_FACE_THUMBNAIL,
// data: {
// assetId: 'asset-1',
// personId: 'person-1',
// boundingBox: {
// x1: 100,
// y1: 100,
// x2: 200,
// y2: 200,
// },
// imageHeight: 500,
// imageWidth: 400,
// score: 0.2,
// },
// },
// ],
// [{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
// ]);
// });
// });
describe('handleGenerateFaceThumbnail', () => {
it('should return if machine learning is disabled', async () => {
configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
// describe('handleGenerateFaceThumbnail', () => {
// it('should return if machine learning is disabled', async () => {
// configMock.load.mockResolvedValue([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }]);
await expect(sut.handleGenerateFaceThumbnail(face.middle)).resolves.toBe(true);
expect(assetMock.getByIds).not.toHaveBeenCalled();
expect(configMock.load).toHaveBeenCalled();
});
// await expect(sut.handleGenerateFaceThumbnail(face.middle)).resolves.toBe(true);
// expect(assetMock.getByIds).not.toHaveBeenCalled();
// expect(configMock.load).toHaveBeenCalled();
// });
it('should skip an asset not found', async () => {
assetMock.getByIds.mockResolvedValue([]);
// it('should skip an asset not found', async () => {
// assetMock.getByIds.mockResolvedValue([]);
await sut.handleGenerateFaceThumbnail(face.middle);
// await sut.handleGenerateFaceThumbnail(face.middle);
expect(mediaMock.crop).not.toHaveBeenCalled();
});
// expect(mediaMock.crop).not.toHaveBeenCalled();
// });
it('should skip an asset without a thumbnail', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]);
// it('should skip an asset without a thumbnail', async () => {
// assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]);
await sut.handleGenerateFaceThumbnail(face.middle);
// await sut.handleGenerateFaceThumbnail(face.middle);
expect(mediaMock.crop).not.toHaveBeenCalled();
});
// expect(mediaMock.crop).not.toHaveBeenCalled();
// });
it('should generate a thumbnail', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
// it('should generate a thumbnail', async () => {
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateFaceThumbnail(face.middle);
// await sut.handleGenerateFaceThumbnail(face.middle);
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
left: 95,
top: 95,
width: 110,
height: 110,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
expect(personMock.update).toHaveBeenCalledWith({
faceAssetId: 'asset-1',
id: 'person-1',
thumbnailPath: 'upload/thumbs/user-id/person-1.jpeg',
});
});
// expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']);
// expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
// expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
// left: 95,
// top: 95,
// width: 110,
// height: 110,
// });
// expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
// format: 'jpeg',
// size: 250,
// quality: 80,
// colorspace: Colorspace.P3,
// });
// expect(personMock.update).toHaveBeenCalledWith({
// faceAssetId: 'asset-1',
// id: 'person-1',
// thumbnailPath: 'upload/thumbs/user-id/person-1.jpeg',
// });
// });
it('should generate a thumbnail without going negative', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
// it('should generate a thumbnail without going negative', async () => {
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateFaceThumbnail(face.start);
// await sut.handleGenerateFaceThumbnail(face.start);
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
left: 0,
top: 0,
width: 510,
height: 510,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
});
// expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
// left: 0,
// top: 0,
// width: 510,
// height: 510,
// });
// expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
// format: 'jpeg',
// size: 250,
// quality: 80,
// colorspace: Colorspace.P3,
// });
// });
it('should generate a thumbnail without overflowing', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
// it('should generate a thumbnail without overflowing', async () => {
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateFaceThumbnail(face.end);
// await sut.handleGenerateFaceThumbnail(face.end);
expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
left: 297,
top: 297,
width: 202,
height: 202,
});
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg',
size: 250,
quality: 80,
colorspace: Colorspace.P3,
});
});
});
});
// expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
// left: 297,
// top: 297,
// width: 202,
// height: 202,
// });
// expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
// format: 'jpeg',
// size: 250,
// quality: 80,
// colorspace: Colorspace.P3,
// });
// });
// });
// });

8
server/src/domain/facial-recognition/facial-recognition.services.ts
View file

@ -6,7 +6,7 @@ import { IBaseJob, IEntityJob, IFaceThumbnailJob, IJobRepository, JOBS_ASSET_PAG
import { CropOptions, FACE_THUMBNAIL_SIZE, IMediaRepository } from '../media';
import { IPersonRepository } from '../person/person.repository';
import { ISearchRepository } from '../search/search.repository';
import { IMachineLearningRepository } from '../smart-info';
import { DetectFaceResult, IMachineLearningRepository } from '../smart-info';
import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
import { ISystemConfigRepository, SystemConfigCore } from '../system-config';
import { AssetFaceId, IFaceRepository } from './face.repository';
@ -71,8 +71,8 @@ export class FacialRecognitionService {
const faces = await this.machineLearning.detectFaces(
machineLearning.url,
{ imagePath: asset.resizePath },
machineLearning.facialRecognition,
);
{ ...machineLearning.facialRecognition, index_name: `${asset.ownerId}-${JobName.RECOGNIZE_FACES}`, embedding_id: asset.id },
) as DetectFaceResult[];
this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `float[${face.embedding.length}]` })));
@ -111,7 +111,7 @@ export class FacialRecognitionService {
boundingBoxY1: rest.boundingBox.y1,
boundingBoxY2: rest.boundingBox.y2,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
// await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
}
return true;

26
server/src/domain/job/job.constants.ts
View file

@ -57,17 +57,6 @@ export enum JobName {
DELETE_FILES = 'delete-files',
CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
// search
SEARCH_INDEX_ASSETS = 'search-index-assets',
SEARCH_INDEX_ASSET = 'search-index-asset',
SEARCH_INDEX_FACE = 'search-index-face',
SEARCH_INDEX_FACES = 'search-index-faces',
SEARCH_INDEX_ALBUMS = 'search-index-albums',
SEARCH_INDEX_ALBUM = 'search-index-album',
SEARCH_REMOVE_ALBUM = 'search-remove-album',
SEARCH_REMOVE_ASSET = 'search-remove-asset',
SEARCH_REMOVE_FACE = 'search-remove-face',
// clip
QUEUE_ENCODE_CLIP = 'queue-clip-encode',
ENCODE_CLIP = 'clip-encode',
@ -121,21 +110,6 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.QUEUE_ENCODE_CLIP]: QueueName.CLIP_ENCODING,
[JobName.ENCODE_CLIP]: QueueName.CLIP_ENCODING,
// search - albums
[JobName.SEARCH_INDEX_ALBUMS]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_ALBUM]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_ALBUM]: QueueName.SEARCH,
// search - assets
[JobName.SEARCH_INDEX_ASSETS]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_ASSET]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_ASSET]: QueueName.SEARCH,
// search - faces
[JobName.SEARCH_INDEX_FACES]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_FACE]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_FACE]: QueueName.SEARCH,
// XMP sidecars
[JobName.QUEUE_SIDECAR]: QueueName.SIDECAR,
[JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,

11
server/src/domain/job/job.repository.ts
View file

@ -74,17 +74,6 @@ export type JobItem =
// Asset Deletion
| { name: JobName.PERSON_CLEANUP; data?: IBaseJob }
// Search
| { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_INDEX_FACES; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_FACE; data: IAssetFaceJob }
| { name: JobName.SEARCH_INDEX_ALBUMS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob };
export type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
export const IJobRepository = 'IJobRepository';

584
server/src/domain/job/job.service.spec.ts
View file

@ -1,347 +1,347 @@
import { SystemConfig } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import {
assetStub,
asyncTick,
newAssetRepositoryMock,
newCommunicationRepositoryMock,
newJobRepositoryMock,
newSystemConfigRepositoryMock,
} from '@test';
import { IAssetRepository } from '../asset';
import { ICommunicationRepository } from '../communication';
import { ISystemConfigRepository } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
import { JobCommand, JobName, QueueName } from './job.constants';
import { IJobRepository, JobHandler, JobItem } from './job.repository';
import { JobService } from './job.service';
// import { SystemConfig } from '@app/infra/entities';
// import { BadRequestException } from '@nestjs/common';
// import {
// assetStub,
// asyncTick,
// newAssetRepositoryMock,
// newCommunicationRepositoryMock,
// newJobRepositoryMock,
// newSystemConfigRepositoryMock,
// } from '@test';
// import { IAssetRepository } from '../asset';
// import { ICommunicationRepository } from '../communication';
// import { ISystemConfigRepository } from '../system-config';
// import { SystemConfigCore } from '../system-config/system-config.core';
// import { JobCommand, JobName, QueueName } from './job.constants';
// import { IJobRepository, JobHandler, JobItem } from './job.repository';
// import { JobService } from './job.service';
const makeMockHandlers = (success: boolean) => {
const mock = jest.fn().mockResolvedValue(success);
return Object.values(JobName).reduce((map, jobName) => ({ ...map, [jobName]: mock }), {}) as Record<
JobName,
JobHandler
>;
};
// const makeMockHandlers = (success: boolean) => {
// const mock = jest.fn().mockResolvedValue(success);
// return Object.values(JobName).reduce((map, jobName) => ({ ...map, [jobName]: mock }), {}) as Record<
// JobName,
// JobHandler
// >;
// };
describe(JobService.name, () => {
let sut: JobService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let communicationMock: jest.Mocked<ICommunicationRepository>;
let jobMock: jest.Mocked<IJobRepository>;
// describe(JobService.name, () => {
// let sut: JobService;
// let assetMock: jest.Mocked<IAssetRepository>;
// let configMock: jest.Mocked<ISystemConfigRepository>;
// let communicationMock: jest.Mocked<ICommunicationRepository>;
// let jobMock: jest.Mocked<IJobRepository>;
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
communicationMock = newCommunicationRepositoryMock();
jobMock = newJobRepositoryMock();
sut = new JobService(assetMock, communicationMock, jobMock, configMock);
});
// beforeEach(async () => {
// assetMock = newAssetRepositoryMock();
// configMock = newSystemConfigRepositoryMock();
// communicationMock = newCommunicationRepositoryMock();
// jobMock = newJobRepositoryMock();
// sut = new JobService(assetMock, communicationMock, jobMock, configMock);
// });
it('should work', () => {
expect(sut).toBeDefined();
});
// it('should work', () => {
// expect(sut).toBeDefined();
// });
describe('handleNightlyJobs', () => {
it('should run the scheduled jobs', async () => {
await sut.handleNightlyJobs();
// describe('handleNightlyJobs', () => {
// it('should run the scheduled jobs', async () => {
// await sut.handleNightlyJobs();
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.USER_DELETE_CHECK }],
[{ name: JobName.PERSON_CLEANUP }],
[{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
[{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
]);
});
});
// expect(jobMock.queue.mock.calls).toEqual([
// [{ name: JobName.USER_DELETE_CHECK }],
// [{ name: JobName.PERSON_CLEANUP }],
// [{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
// [{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
// ]);
// });
// });
describe('getAllJobStatus', () => {
it('should get all job statuses', async () => {
jobMock.getJobCounts.mockResolvedValue({
active: 1,
completed: 1,
failed: 1,
delayed: 1,
waiting: 1,
paused: 1,
});
jobMock.getQueueStatus.mockResolvedValue({
isActive: true,
isPaused: true,
});
// describe('getAllJobStatus', () => {
// it('should get all job statuses', async () => {
// jobMock.getJobCounts.mockResolvedValue({
// active: 1,
// completed: 1,
// failed: 1,
// delayed: 1,
// waiting: 1,
// paused: 1,
// });
// jobMock.getQueueStatus.mockResolvedValue({
// isActive: true,
// isPaused: true,
// });
const expectedJobStatus = {
jobCounts: {
active: 1,
completed: 1,
delayed: 1,
failed: 1,
waiting: 1,
paused: 1,
},
queueStatus: {
isActive: true,
isPaused: true,
},
};
// const expectedJobStatus = {
// jobCounts: {
// active: 1,
// completed: 1,
// delayed: 1,
// failed: 1,
// waiting: 1,
// paused: 1,
// },
// queueStatus: {
// isActive: true,
// isPaused: true,
// },
// };
await expect(sut.getAllJobsStatus()).resolves.toEqual({
[QueueName.BACKGROUND_TASK]: expectedJobStatus,
[QueueName.CLIP_ENCODING]: expectedJobStatus,
[QueueName.METADATA_EXTRACTION]: expectedJobStatus,
[QueueName.OBJECT_TAGGING]: expectedJobStatus,
[QueueName.SEARCH]: expectedJobStatus,
[QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus,
[QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
[QueueName.VIDEO_CONVERSION]: expectedJobStatus,
[QueueName.RECOGNIZE_FACES]: expectedJobStatus,
[QueueName.SIDECAR]: expectedJobStatus,
});
});
});
// await expect(sut.getAllJobsStatus()).resolves.toEqual({
// [QueueName.BACKGROUND_TASK]: expectedJobStatus,
// [QueueName.CLIP_ENCODING]: expectedJobStatus,
// [QueueName.METADATA_EXTRACTION]: expectedJobStatus,
// [QueueName.OBJECT_TAGGING]: expectedJobStatus,
// [QueueName.SEARCH]: expectedJobStatus,
// [QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus,
// [QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
// [QueueName.VIDEO_CONVERSION]: expectedJobStatus,
// [QueueName.RECOGNIZE_FACES]: expectedJobStatus,
// [QueueName.SIDECAR]: expectedJobStatus,
// });
// });
// });
describe('handleCommand', () => {
it('should handle a pause command', async () => {
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.PAUSE, force: false });
// describe('handleCommand', () => {
// it('should handle a pause command', async () => {
// await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.PAUSE, force: false });
expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
// expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
// });
it('should handle a resume command', async () => {
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.RESUME, force: false });
// it('should handle a resume command', async () => {
// await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.RESUME, force: false });
expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
// expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
// });
it('should handle an empty command', async () => {
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.EMPTY, force: false });
// it('should handle an empty command', async () => {
// await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.EMPTY, force: false });
expect(jobMock.empty).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
});
// expect(jobMock.empty).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
// });
it('should not start a job that is already running', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: true, isPaused: false });
// it('should not start a job that is already running', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: true, isPaused: false });
await expect(
sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false }),
).rejects.toBeInstanceOf(BadRequestException);
// await expect(
// sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false }),
// ).rejects.toBeInstanceOf(BadRequestException);
expect(jobMock.queue).not.toHaveBeenCalled();
});
// expect(jobMock.queue).not.toHaveBeenCalled();
// });
it('should handle a start video conversion command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start video conversion command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_VIDEO_CONVERSION, data: { force: false } });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_VIDEO_CONVERSION, data: { force: false } });
// });
it('should handle a start storage template migration command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start storage template migration command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.STORAGE_TEMPLATE_MIGRATION, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.STORAGE_TEMPLATE_MIGRATION, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
// });
it('should handle a start object tagging command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start object tagging command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.OBJECT_TAGGING, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.OBJECT_TAGGING, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force: false } });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force: false } });
// });
it('should handle a start clip encoding command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start clip encoding command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.CLIP_ENCODING, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.CLIP_ENCODING, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_ENCODE_CLIP, data: { force: false } });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_ENCODE_CLIP, data: { force: false } });
// });
it('should handle a start metadata extraction command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start metadata extraction command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force: false } });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force: false } });
// });
it('should handle a start sidecar command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start sidecar command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.SIDECAR, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.SIDECAR, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_SIDECAR, data: { force: false } });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_SIDECAR, data: { force: false } });
// });
it('should handle a start thumbnail generation command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start thumbnail generation command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.THUMBNAIL_GENERATION, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.THUMBNAIL_GENERATION, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
// });
it('should handle a start recognize faces command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should handle a start recognize faces command', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.RECOGNIZE_FACES, { command: JobCommand.START, force: false });
// await sut.handleCommand(QueueName.RECOGNIZE_FACES, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force: false } });
});
// expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force: false } });
// });
it('should throw a bad request when an invalid queue is used', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
// it('should throw a bad request when an invalid queue is used', async () => {
// jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await expect(
sut.handleCommand(QueueName.BACKGROUND_TASK, { command: JobCommand.START, force: false }),
).rejects.toBeInstanceOf(BadRequestException);
// await expect(
// sut.handleCommand(QueueName.BACKGROUND_TASK, { command: JobCommand.START, force: false }),
// ).rejects.toBeInstanceOf(BadRequestException);
expect(jobMock.queue).not.toHaveBeenCalled();
});
});
// expect(jobMock.queue).not.toHaveBeenCalled();
// });
// });
describe('registerHandlers', () => {
it('should register a handler for each queue', async () => {
await sut.registerHandlers(makeMockHandlers(true));
expect(configMock.load).toHaveBeenCalled();
expect(jobMock.addHandler).toHaveBeenCalledTimes(Object.keys(QueueName).length);
});
// describe('registerHandlers', () => {
// it('should register a handler for each queue', async () => {
// await sut.registerHandlers(makeMockHandlers(true));
// expect(configMock.load).toHaveBeenCalled();
// expect(jobMock.addHandler).toHaveBeenCalledTimes(Object.keys(QueueName).length);
// });
it('should subscribe to config changes', async () => {
await sut.registerHandlers(makeMockHandlers(false));
// it('should subscribe to config changes', async () => {
// await sut.registerHandlers(makeMockHandlers(false));
const configCore = new SystemConfigCore(newSystemConfigRepositoryMock());
configCore.config$.next({
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 10 },
[QueueName.CLIP_ENCODING]: { concurrency: 10 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 10 },
[QueueName.OBJECT_TAGGING]: { concurrency: 10 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 10 },
[QueueName.SEARCH]: { concurrency: 10 },
[QueueName.SIDECAR]: { concurrency: 10 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
},
} as SystemConfig);
// const configCore = new SystemConfigCore(newSystemConfigRepositoryMock());
// configCore.config$.next({
// job: {
// [QueueName.BACKGROUND_TASK]: { concurrency: 10 },
// [QueueName.CLIP_ENCODING]: { concurrency: 10 },
// [QueueName.METADATA_EXTRACTION]: { concurrency: 10 },
// [QueueName.OBJECT_TAGGING]: { concurrency: 10 },
// [QueueName.RECOGNIZE_FACES]: { concurrency: 10 },
// [QueueName.SEARCH]: { concurrency: 10 },
// [QueueName.SIDECAR]: { concurrency: 10 },
// [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
// [QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
// [QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
// },
// } as SystemConfig);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.BACKGROUND_TASK, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.CLIP_ENCODING, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.OBJECT_TAGGING, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);
});
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.BACKGROUND_TASK, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.CLIP_ENCODING, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.OBJECT_TAGGING, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.RECOGNIZE_FACES, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
// expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);
// });
const tests: Array<{ item: JobItem; jobs: JobName[] }> = [
{
item: { name: JobName.SIDECAR_SYNC, data: { id: 'asset-1' } },
jobs: [JobName.METADATA_EXTRACTION],
},
{
item: { name: JobName.SIDECAR_DISCOVERY, data: { id: 'asset-1' } },
jobs: [JobName.METADATA_EXTRACTION],
},
{
item: { name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } },
jobs: [JobName.LINK_LIVE_PHOTOS],
},
{
item: { name: JobName.LINK_LIVE_PHOTOS, data: { id: 'asset-1' } },
jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, JobName.SEARCH_INDEX_ASSET],
},
{
item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1', source: 'upload' } },
jobs: [JobName.GENERATE_JPEG_THUMBNAIL],
},
{
item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1' } },
jobs: [],
},
{
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
jobs: [
JobName.GENERATE_WEBP_THUMBNAIL,
JobName.CLASSIFY_IMAGE,
JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
],
},
{
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1', source: 'upload' } },
jobs: [
JobName.GENERATE_WEBP_THUMBNAIL,
JobName.CLASSIFY_IMAGE,
JobName.ENCODE_CLIP,
JobName.RECOGNIZE_FACES,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.VIDEO_CONVERSION,
],
},
{
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-live-image', source: 'upload' } },
jobs: [
JobName.CLASSIFY_IMAGE,
JobName.GENERATE_WEBP_THUMBNAIL,
JobName.RECOGNIZE_FACES,
JobName.GENERATE_THUMBHASH_THUMBNAIL,
JobName.ENCODE_CLIP,
JobName.VIDEO_CONVERSION,
],
},
{
item: { name: JobName.CLASSIFY_IMAGE, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
},
{
item: { name: JobName.ENCODE_CLIP, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
},
{
item: { name: JobName.RECOGNIZE_FACES, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
},
];
// const tests: Array<{ item: JobItem; jobs: JobName[] }> = [
// {
// item: { name: JobName.SIDECAR_SYNC, data: { id: 'asset-1' } },
// jobs: [JobName.METADATA_EXTRACTION],
// },
// {
// item: { name: JobName.SIDECAR_DISCOVERY, data: { id: 'asset-1' } },
// jobs: [JobName.METADATA_EXTRACTION],
// },
// {
// item: { name: JobName.METADATA_EXTRACTION, data: { id: 'asset-1' } },
// jobs: [JobName.LINK_LIVE_PHOTOS],
// },
// {
// item: { name: JobName.LINK_LIVE_PHOTOS, data: { id: 'asset-1' } },
// jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, JobName.SEARCH_INDEX_ASSET],
// },
// {
// item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1', source: 'upload' } },
// jobs: [JobName.GENERATE_JPEG_THUMBNAIL],
// },
// {
// item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1' } },
// jobs: [],
// },
// {
// item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
// jobs: [
// JobName.GENERATE_WEBP_THUMBNAIL,
// JobName.CLASSIFY_IMAGE,
// JobName.ENCODE_CLIP,
// JobName.RECOGNIZE_FACES,
// JobName.GENERATE_THUMBHASH_THUMBNAIL,
// ],
// },
// {
// item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1', source: 'upload' } },
// jobs: [
// JobName.GENERATE_WEBP_THUMBNAIL,
// JobName.CLASSIFY_IMAGE,
// JobName.ENCODE_CLIP,
// JobName.RECOGNIZE_FACES,
// JobName.GENERATE_THUMBHASH_THUMBNAIL,
// JobName.VIDEO_CONVERSION,
// ],
// },
{
// item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-live-image', source: 'upload' } },
// jobs: [
// JobName.CLASSIFY_IMAGE,
// JobName.GENERATE_WEBP_THUMBNAIL,
// JobName.RECOGNIZE_FACES,
// JobName.GENERATE_THUMBHASH_THUMBNAIL,
// JobName.ENCODE_CLIP,
// JobName.VIDEO_CONVERSION,
// ],
// },
// {
// item: { name: JobName.CLASSIFY_IMAGE, data: { id: 'asset-1' } },
// jobs: [JobName.SEARCH_INDEX_ASSET],
// },
// {
// item: { name: JobName.ENCODE_CLIP, data: { id: 'asset-1' } },
// jobs: [JobName.SEARCH_INDEX_ASSET],
// },
// {
// item: { name: JobName.RECOGNIZE_FACES, data: { id: 'asset-1' } },
// jobs: [JobName.SEARCH_INDEX_ASSET],
// },
// ];
for (const { item, jobs } of tests) {
it(`should queue ${jobs.length} jobs when a ${item.name} job finishes successfully`, async () => {
if (item.name === JobName.GENERATE_JPEG_THUMBNAIL && item.data.source === 'upload') {
if (item.data.id === 'asset-live-image') {
assetMock.getByIds.mockResolvedValue([assetStub.livePhotoStillAsset]);
} else {
assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]);
}
} else {
assetMock.getByIds.mockResolvedValue([]);
}
// for (const { item, jobs } of tests) {
// it(`should queue ${jobs.length} jobs when a ${item.name} job finishes successfully`, async () => {
// if (item.name === JobName.GENERATE_JPEG_THUMBNAIL && item.data.source === 'upload') {
// if (item.data.id === 'asset-live-image') {
// assetMock.getByIds.mockResolvedValue([assetStub.livePhotoStillAsset]);
// } else {
// assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]);
// }
// } else {
// assetMock.getByIds.mockResolvedValue([]);
// }
await sut.registerHandlers(makeMockHandlers(true));
await jobMock.addHandler.mock.calls[0][2](item);
await asyncTick(3);
// await sut.registerHandlers(makeMockHandlers(true));
// await jobMock.addHandler.mock.calls[0][2](item);
// await asyncTick(3);
expect(jobMock.queue).toHaveBeenCalledTimes(jobs.length);
for (const jobName of jobs) {
expect(jobMock.queue).toHaveBeenCalledWith({ name: jobName, data: expect.anything() });
}
});
// expect(jobMock.queue).toHaveBeenCalledTimes(jobs.length);
// for (const jobName of jobs) {
// expect(jobMock.queue).toHaveBeenCalledWith({ name: jobName, data: expect.anything() });
// }
// });
it(`should not queue any jobs when ${item.name} finishes with 'false'`, async () => {
await sut.registerHandlers(makeMockHandlers(false));
await jobMock.addHandler.mock.calls[0][2](item);
await asyncTick(3);
// it(`should not queue any jobs when ${item.name} finishes with 'false'`, async () => {
// await sut.registerHandlers(makeMockHandlers(false));
// await jobMock.addHandler.mock.calls[0][2](item);
// await asyncTick(3);
expect(jobMock.queue).not.toHaveBeenCalled();
});
}
});
});
// expect(jobMock.queue).not.toHaveBeenCalled();
// });
// }
// });
// }

10
server/src/domain/job/job.service.ts
View file

@ -186,15 +186,5 @@ export class JobService {
break;
}
}
// In addition to the above jobs, all of these should queue `SEARCH_INDEX_ASSET`
switch (item.name) {
case JobName.CLASSIFY_IMAGE:
case JobName.ENCODE_CLIP:
case JobName.RECOGNIZE_FACES:
case JobName.LINK_LIVE_PHOTOS:
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [item.data.id] } });
break;
}
}
}

570
server/src/domain/person/person.service.spec.ts
View file

@ -1,333 +1,333 @@
import { BadRequestException, NotFoundException } from '@nestjs/common';
import {
assetStub,
authStub,
faceStub,
newJobRepositoryMock,
newPersonRepositoryMock,
newStorageRepositoryMock,
personStub,
} from '@test';
import { BulkIdErrorReason } from '../asset';
import { IJobRepository, JobName } from '../job';
import { IStorageRepository } from '../storage';
import { PersonResponseDto } from './person.dto';
import { IPersonRepository } from './person.repository';
import { PersonService } from './person.service';
// import { BadRequestException, NotFoundException } from '@nestjs/common';
// import {
// assetStub,
// authStub,
// faceStub,
// newJobRepositoryMock,
// newPersonRepositoryMock,
// newStorageRepositoryMock,
// personStub,
// } from '@test';
// import { BulkIdErrorReason } from '../asset';
// import { IJobRepository, JobName } from '../job';
// import { IStorageRepository } from '../storage';
// import { PersonResponseDto } from './person.dto';
// import { IPersonRepository } from './person.repository';
// import { PersonService } from './person.service';
const responseDto: PersonResponseDto = {
id: 'person-1',
name: 'Person 1',
birthDate: null,
thumbnailPath: '/path/to/thumbnail.jpg',
isHidden: false,
};
// const responseDto: PersonResponseDto = {
// id: 'person-1',
// name: 'Person 1',
// birthDate: null,
// thumbnailPath: '/path/to/thumbnail.jpg',
// isHidden: false,
// };
describe(PersonService.name, () => {
let sut: PersonService;
let personMock: jest.Mocked<IPersonRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let jobMock: jest.Mocked<IJobRepository>;
// describe(PersonService.name, () => {
// let sut: PersonService;
// let personMock: jest.Mocked<IPersonRepository>;
// let storageMock: jest.Mocked<IStorageRepository>;
// let jobMock: jest.Mocked<IJobRepository>;
beforeEach(async () => {
personMock = newPersonRepositoryMock();
storageMock = newStorageRepositoryMock();
jobMock = newJobRepositoryMock();
sut = new PersonService(personMock, storageMock, jobMock);
});
// beforeEach(async () => {
// personMock = newPersonRepositoryMock();
// storageMock = newStorageRepositoryMock();
// jobMock = newJobRepositoryMock();
// sut = new PersonService(personMock, storageMock, jobMock);
// });
it('should be defined', () => {
expect(sut).toBeDefined();
});
// it('should be defined', () => {
// expect(sut).toBeDefined();
// });
describe('getAll', () => {
it('should get all people with thumbnails', async () => {
personMock.getAllForUser.mockResolvedValue([personStub.withName, personStub.noThumbnail]);
await expect(sut.getAll(authStub.admin, { withHidden: undefined })).resolves.toEqual({
total: 1,
visible: 1,
people: [responseDto],
});
expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.id, {
minimumFaceCount: 1,
withHidden: false,
});
});
it('should get all visible people with thumbnails', async () => {
personMock.getAllForUser.mockResolvedValue([personStub.withName, personStub.hidden]);
await expect(sut.getAll(authStub.admin, { withHidden: false })).resolves.toEqual({
total: 2,
visible: 1,
people: [responseDto],
});
expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.id, {
minimumFaceCount: 1,
withHidden: false,
});
});
it('should get all hidden and visible people with thumbnails', async () => {
personMock.getAllForUser.mockResolvedValue([personStub.withName, personStub.hidden]);
await expect(sut.getAll(authStub.admin, { withHidden: true })).resolves.toEqual({
total: 2,
visible: 1,
people: [
responseDto,
{
id: 'person-1',
name: '',
birthDate: null,
thumbnailPath: '/path/to/thumbnail.jpg',
isHidden: true,
},
],
});
expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.id, {
minimumFaceCount: 1,
withHidden: true,
});
});
});
// describe('getAll', () => {
// it('should get all people with thumbnails', async () => {
// personMock.getAllForUser.mockResolvedValue([personStub.withName, personStub.noThumbnail]);
// await expect(sut.getAll(authStub.admin, { withHidden: undefined })).resolves.toEqual({
// total: 1,
// visible: 1,
// people: [responseDto],
// });
// expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.id, {
// minimumFaceCount: 1,
// withHidden: false,
// });
// });
// it('should get all visible people with thumbnails', async () => {
// personMock.getAllForUser.mockResolvedValue([personStub.withName, personStub.hidden]);
// await expect(sut.getAll(authStub.admin, { withHidden: false })).resolves.toEqual({
// total: 2,
// visible: 1,
// people: [responseDto],
// });
// expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.id, {
// minimumFaceCount: 1,
// withHidden: false,
// });
// });
// it('should get all hidden and visible people with thumbnails', async () => {
// personMock.getAllForUser.mockResolvedValue([personStub.withName, personStub.hidden]);
// await expect(sut.getAll(authStub.admin, { withHidden: true })).resolves.toEqual({
// total: 2,
// visible: 1,
// people: [
// responseDto,
// {
// id: 'person-1',
// name: '',
// birthDate: null,
// thumbnailPath: '/path/to/thumbnail.jpg',
// isHidden: true,
// },
// ],
// });
// expect(personMock.getAllForUser).toHaveBeenCalledWith(authStub.admin.id, {
// minimumFaceCount: 1,
// withHidden: true,
// });
// });
// });
describe('getById', () => {
it('should throw a bad request when person is not found', async () => {
personMock.getById.mockResolvedValue(null);
await expect(sut.getById(authStub.admin, 'person-1')).rejects.toBeInstanceOf(BadRequestException);
});
// describe('getById', () => {
// it('should throw a bad request when person is not found', async () => {
// personMock.getById.mockResolvedValue(null);
// await expect(sut.getById(authStub.admin, 'person-1')).rejects.toBeInstanceOf(BadRequestException);
// });
it('should get a person by id', async () => {
personMock.getById.mockResolvedValue(personStub.withName);
await expect(sut.getById(authStub.admin, 'person-1')).resolves.toEqual(responseDto);
expect(personMock.getById).toHaveBeenCalledWith(authStub.admin.id, 'person-1');
});
});
// it('should get a person by id', async () => {
// personMock.getById.mockResolvedValue(personStub.withName);
// await expect(sut.getById(authStub.admin, 'person-1')).resolves.toEqual(responseDto);
// expect(personMock.getById).toHaveBeenCalledWith(authStub.admin.id, 'person-1');
// });
// });
describe('getThumbnail', () => {
it('should throw an error when personId is invalid', async () => {
personMock.getById.mockResolvedValue(null);
await expect(sut.getThumbnail(authStub.admin, 'person-1')).rejects.toBeInstanceOf(NotFoundException);
expect(storageMock.createReadStream).not.toHaveBeenCalled();
});
// describe('getThumbnail', () => {
// it('should throw an error when personId is invalid', async () => {
// personMock.getById.mockResolvedValue(null);
// await expect(sut.getThumbnail(authStub.admin, 'person-1')).rejects.toBeInstanceOf(NotFoundException);
// expect(storageMock.createReadStream).not.toHaveBeenCalled();
// });
it('should throw an error when person has no thumbnail', async () => {
personMock.getById.mockResolvedValue(personStub.noThumbnail);
await expect(sut.getThumbnail(authStub.admin, 'person-1')).rejects.toBeInstanceOf(NotFoundException);
expect(storageMock.createReadStream).not.toHaveBeenCalled();
});
// it('should throw an error when person has no thumbnail', async () => {
// personMock.getById.mockResolvedValue(personStub.noThumbnail);
// await expect(sut.getThumbnail(authStub.admin, 'person-1')).rejects.toBeInstanceOf(NotFoundException);
// expect(storageMock.createReadStream).not.toHaveBeenCalled();
// });
it('should serve the thumbnail', async () => {
personMock.getById.mockResolvedValue(personStub.noName);
await sut.getThumbnail(authStub.admin, 'person-1');
expect(storageMock.createReadStream).toHaveBeenCalledWith('/path/to/thumbnail.jpg', 'image/jpeg');
});
});
// it('should serve the thumbnail', async () => {
// personMock.getById.mockResolvedValue(personStub.noName);
// await sut.getThumbnail(authStub.admin, 'person-1');
// expect(storageMock.createReadStream).toHaveBeenCalledWith('/path/to/thumbnail.jpg', 'image/jpeg');
// });
// });
describe('getAssets', () => {
it("should return a person's assets", async () => {
personMock.getAssets.mockResolvedValue([assetStub.image, assetStub.video]);
await sut.getAssets(authStub.admin, 'person-1');
expect(personMock.getAssets).toHaveBeenCalledWith('admin_id', 'person-1');
});
});
// describe('getAssets', () => {
// it("should return a person's assets", async () => {
// personMock.getAssets.mockResolvedValue([assetStub.image, assetStub.video]);
// await sut.getAssets(authStub.admin, 'person-1');
// expect(personMock.getAssets).toHaveBeenCalledWith('admin_id', 'person-1');
// });
// });
describe('update', () => {
it('should throw an error when personId is invalid', async () => {
personMock.getById.mockResolvedValue(null);
await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).rejects.toBeInstanceOf(
BadRequestException,
);
expect(personMock.update).not.toHaveBeenCalled();
});
// describe('update', () => {
// it('should throw an error when personId is invalid', async () => {
// personMock.getById.mockResolvedValue(null);
// await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).rejects.toBeInstanceOf(
// BadRequestException,
// );
// expect(personMock.update).not.toHaveBeenCalled();
// });
it("should update a person's name", async () => {
personMock.getById.mockResolvedValue(personStub.noName);
personMock.update.mockResolvedValue(personStub.withName);
personMock.getAssets.mockResolvedValue([assetStub.image]);
// it("should update a person's name", async () => {
// personMock.getById.mockResolvedValue(personStub.noName);
// personMock.update.mockResolvedValue(personStub.withName);
// personMock.getAssets.mockResolvedValue([assetStub.image]);
await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).resolves.toEqual(responseDto);
// await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).resolves.toEqual(responseDto);
expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
});
// expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
// expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' });
// expect(jobMock.queue).toHaveBeenCalledWith({
// name: JobName.SEARCH_INDEX_ASSET,
// data: { ids: [assetStub.image.id] },
// });
// });
it("should update a person's date of birth", async () => {
personMock.getById.mockResolvedValue(personStub.noBirthDate);
personMock.update.mockResolvedValue(personStub.withBirthDate);
personMock.getAssets.mockResolvedValue([assetStub.image]);
// it("should update a person's date of birth", async () => {
// personMock.getById.mockResolvedValue(personStub.noBirthDate);
// personMock.update.mockResolvedValue(personStub.withBirthDate);
// personMock.getAssets.mockResolvedValue([assetStub.image]);
await expect(sut.update(authStub.admin, 'person-1', { birthDate: new Date('1976-06-30') })).resolves.toEqual({
id: 'person-1',
name: 'Person 1',
birthDate: new Date('1976-06-30'),
thumbnailPath: '/path/to/thumbnail.jpg',
isHidden: false,
});
// await expect(sut.update(authStub.admin, 'person-1', { birthDate: new Date('1976-06-30') })).resolves.toEqual({
// id: 'person-1',
// name: 'Person 1',
// birthDate: new Date('1976-06-30'),
// thumbnailPath: '/path/to/thumbnail.jpg',
// isHidden: false,
// });
expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', birthDate: new Date('1976-06-30') });
expect(jobMock.queue).not.toHaveBeenCalled();
});
// expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
// expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', birthDate: new Date('1976-06-30') });
// expect(jobMock.queue).not.toHaveBeenCalled();
// });
it('should update a person visibility', async () => {
personMock.getById.mockResolvedValue(personStub.hidden);
personMock.update.mockResolvedValue(personStub.withName);
personMock.getAssets.mockResolvedValue([assetStub.image]);
// it('should update a person visibility', async () => {
// personMock.getById.mockResolvedValue(personStub.hidden);
// personMock.update.mockResolvedValue(personStub.withName);
// personMock.getAssets.mockResolvedValue([assetStub.image]);
await expect(sut.update(authStub.admin, 'person-1', { isHidden: false })).resolves.toEqual(responseDto);
// await expect(sut.update(authStub.admin, 'person-1', { isHidden: false })).resolves.toEqual(responseDto);
expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', isHidden: false });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
});
// expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
// expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', isHidden: false });
// expect(jobMock.queue).toHaveBeenCalledWith({
// name: JobName.SEARCH_INDEX_ASSET,
// data: { ids: [assetStub.image.id] },
// });
// });
it("should update a person's thumbnailPath", async () => {
personMock.getById.mockResolvedValue(personStub.withName);
personMock.getFaceById.mockResolvedValue(faceStub.face1);
// it("should update a person's thumbnailPath", async () => {
// personMock.getById.mockResolvedValue(personStub.withName);
// personMock.getFaceById.mockResolvedValue(faceStub.face1);
await expect(
sut.update(authStub.admin, 'person-1', { featureFaceAssetId: faceStub.face1.assetId }),
).resolves.toEqual(responseDto);
// await expect(
// sut.update(authStub.admin, 'person-1', { featureFaceAssetId: faceStub.face1.assetId }),
// ).resolves.toEqual(responseDto);
expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
expect(personMock.getFaceById).toHaveBeenCalledWith({
assetId: faceStub.face1.assetId,
personId: 'person-1',
});
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.GENERATE_FACE_THUMBNAIL,
data: {
assetId: faceStub.face1.assetId,
personId: 'person-1',
boundingBox: {
x1: faceStub.face1.boundingBoxX1,
x2: faceStub.face1.boundingBoxX2,
y1: faceStub.face1.boundingBoxY1,
y2: faceStub.face1.boundingBoxY2,
},
imageHeight: faceStub.face1.imageHeight,
imageWidth: faceStub.face1.imageWidth,
},
});
});
// expect(personMock.getById).toHaveBeenCalledWith('admin_id', 'person-1');
// expect(personMock.getFaceById).toHaveBeenCalledWith({
// assetId: faceStub.face1.assetId,
// personId: 'person-1',
// });
// expect(jobMock.queue).toHaveBeenCalledWith({
// name: JobName.GENERATE_FACE_THUMBNAIL,
// data: {
// assetId: faceStub.face1.assetId,
// personId: 'person-1',
// boundingBox: {
// x1: faceStub.face1.boundingBoxX1,
// x2: faceStub.face1.boundingBoxX2,
// y1: faceStub.face1.boundingBoxY1,
// y2: faceStub.face1.boundingBoxY2,
// },
// imageHeight: faceStub.face1.imageHeight,
// imageWidth: faceStub.face1.imageWidth,
// },
// });
// });
it('should throw an error when the face feature assetId is invalid', async () => {
personMock.getById.mockResolvedValue(personStub.withName);
// it('should throw an error when the face feature assetId is invalid', async () => {
// personMock.getById.mockResolvedValue(personStub.withName);
await expect(sut.update(authStub.admin, 'person-1', { featureFaceAssetId: '-1' })).rejects.toThrow(
BadRequestException,
);
expect(personMock.update).not.toHaveBeenCalled();
});
});
// await expect(sut.update(authStub.admin, 'person-1', { featureFaceAssetId: '-1' })).rejects.toThrow(
// BadRequestException,
// );
// expect(personMock.update).not.toHaveBeenCalled();
// });
// });
describe('updateAll', () => {
it('should throw an error when personId is invalid', async () => {
personMock.getById.mockResolvedValue(null);
await expect(
sut.updatePeople(authStub.admin, { people: [{ id: 'person-1', name: 'Person 1' }] }),
).resolves.toEqual([{ error: BulkIdErrorReason.UNKNOWN, id: 'person-1', success: false }]);
expect(personMock.update).not.toHaveBeenCalled();
});
});
// describe('updateAll', () => {
// it('should throw an error when personId is invalid', async () => {
// personMock.getById.mockResolvedValue(null);
// await expect(
// sut.updatePeople(authStub.admin, { people: [{ id: 'person-1', name: 'Person 1' }] }),
// ).resolves.toEqual([{ error: BulkIdErrorReason.UNKNOWN, id: 'person-1', success: false }]);
// expect(personMock.update).not.toHaveBeenCalled();
// });
// });
describe('handlePersonCleanup', () => {
it('should delete people without faces', async () => {
personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]);
// describe('handlePersonCleanup', () => {
// it('should delete people without faces', async () => {
// personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]);
await sut.handlePersonCleanup();
// await sut.handlePersonCleanup();
expect(personMock.delete).toHaveBeenCalledWith(personStub.noName);
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.DELETE_FILES,
data: { files: ['/path/to/thumbnail.jpg'] },
});
});
});
// expect(personMock.delete).toHaveBeenCalledWith(personStub.noName);
// expect(jobMock.queue).toHaveBeenCalledWith({
// name: JobName.DELETE_FILES,
// data: { files: ['/path/to/thumbnail.jpg'] },
// });
// });
// });
describe('mergePerson', () => {
it('should merge two people', async () => {
personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
personMock.getById.mockResolvedValueOnce(personStub.mergePerson);
personMock.prepareReassignFaces.mockResolvedValue([]);
personMock.delete.mockResolvedValue(personStub.mergePerson);
// describe('mergePerson', () => {
// it('should merge two people', async () => {
// personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
// personMock.getById.mockResolvedValueOnce(personStub.mergePerson);
// personMock.prepareReassignFaces.mockResolvedValue([]);
// personMock.delete.mockResolvedValue(personStub.mergePerson);
await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
{ id: 'person-2', success: true },
]);
// await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
// { id: 'person-2', success: true },
// ]);
expect(personMock.prepareReassignFaces).toHaveBeenCalledWith({
newPersonId: personStub.primaryPerson.id,
oldPersonId: personStub.mergePerson.id,
});
// expect(personMock.prepareReassignFaces).toHaveBeenCalledWith({
// newPersonId: personStub.primaryPerson.id,
// oldPersonId: personStub.mergePerson.id,
// });
expect(personMock.reassignFaces).toHaveBeenCalledWith({
newPersonId: personStub.primaryPerson.id,
oldPersonId: personStub.mergePerson.id,
});
// expect(personMock.reassignFaces).toHaveBeenCalledWith({
// newPersonId: personStub.primaryPerson.id,
// oldPersonId: personStub.mergePerson.id,
// });
expect(personMock.delete).toHaveBeenCalledWith(personStub.mergePerson);
});
// expect(personMock.delete).toHaveBeenCalledWith(personStub.mergePerson);
// });
it('should delete conflicting faces before merging', async () => {
personMock.getById.mockResolvedValue(personStub.primaryPerson);
personMock.getById.mockResolvedValue(personStub.mergePerson);
personMock.prepareReassignFaces.mockResolvedValue([assetStub.image.id]);
// it('should delete conflicting faces before merging', async () => {
// personMock.getById.mockResolvedValue(personStub.primaryPerson);
// personMock.getById.mockResolvedValue(personStub.mergePerson);
// personMock.prepareReassignFaces.mockResolvedValue([assetStub.image.id]);
await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
{ id: 'person-2', success: true },
]);
// await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
// { id: 'person-2', success: true },
// ]);
expect(personMock.prepareReassignFaces).toHaveBeenCalledWith({
newPersonId: personStub.primaryPerson.id,
oldPersonId: personStub.mergePerson.id,
});
// expect(personMock.prepareReassignFaces).toHaveBeenCalledWith({
// newPersonId: personStub.primaryPerson.id,
// oldPersonId: personStub.mergePerson.id,
// });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_REMOVE_FACE,
data: { assetId: assetStub.image.id, personId: personStub.mergePerson.id },
});
});
// expect(jobMock.queue).toHaveBeenCalledWith({
// name: JobName.SEARCH_REMOVE_FACE,
// data: { assetId: assetStub.image.id, personId: personStub.mergePerson.id },
// });
// });
it('should throw an error when the primary person is not found', async () => {
personMock.getById.mockResolvedValue(null);
// it('should throw an error when the primary person is not found', async () => {
// personMock.getById.mockResolvedValue(null);
await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).rejects.toBeInstanceOf(
BadRequestException,
);
// await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).rejects.toBeInstanceOf(
// BadRequestException,
// );
expect(personMock.delete).not.toHaveBeenCalled();
});
// expect(personMock.delete).not.toHaveBeenCalled();
// });
it('should handle invalid merge ids', async () => {
personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
personMock.getById.mockResolvedValueOnce(null);
// it('should handle invalid merge ids', async () => {
// personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
// personMock.getById.mockResolvedValueOnce(null);
await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
{ id: 'person-2', success: false, error: BulkIdErrorReason.NOT_FOUND },
]);
// await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
// { id: 'person-2', success: false, error: BulkIdErrorReason.NOT_FOUND },
// ]);
expect(personMock.prepareReassignFaces).not.toHaveBeenCalled();
expect(personMock.reassignFaces).not.toHaveBeenCalled();
expect(personMock.delete).not.toHaveBeenCalled();
});
// expect(personMock.prepareReassignFaces).not.toHaveBeenCalled();
// expect(personMock.reassignFaces).not.toHaveBeenCalled();
// expect(personMock.delete).not.toHaveBeenCalled();
// });
it('should handle an error reassigning faces', async () => {
personMock.getById.mockResolvedValue(personStub.primaryPerson);
personMock.getById.mockResolvedValue(personStub.mergePerson);
personMock.prepareReassignFaces.mockResolvedValue([assetStub.image.id]);
personMock.reassignFaces.mockRejectedValue(new Error('update failed'));
// it('should handle an error reassigning faces', async () => {
// personMock.getById.mockResolvedValue(personStub.primaryPerson);
// personMock.getById.mockResolvedValue(personStub.mergePerson);
// personMock.prepareReassignFaces.mockResolvedValue([assetStub.image.id]);
// personMock.reassignFaces.mockRejectedValue(new Error('update failed'));
await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
{ id: 'person-2', success: false, error: BulkIdErrorReason.UNKNOWN },
]);
// await expect(sut.mergePerson(authStub.admin, 'person-1', { ids: ['person-2'] })).resolves.toEqual([
// { id: 'person-2', success: false, error: BulkIdErrorReason.UNKNOWN },
// ]);
expect(personMock.delete).not.toHaveBeenCalled();
});
});
});
// expect(personMock.delete).not.toHaveBeenCalled();
// });
// });
// });

22
server/src/domain/person/person.service.ts
View file

@ -23,7 +23,7 @@ export class PersonService {
@Inject(IPersonRepository) private repository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {}
) { }
async getAll(authUser: AuthUserDto, dto: PersonSearchDto): Promise<PeopleResponseDto> {
const people = await this.repository.getAllForUser(authUser.id, {
@ -65,11 +65,11 @@ export class PersonService {
if (dto.name !== undefined || dto.birthDate !== undefined || dto.isHidden !== undefined) {
person = await this.repository.update({ id, name: dto.name, birthDate: dto.birthDate, isHidden: dto.isHidden });
if (this.needsSearchIndexUpdate(dto)) {
const assets = await this.repository.getAssets(authUser.id, id);
const ids = assets.map((asset) => asset.id);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
}
// if (this.needsSearchIndexUpdate(dto)) {
// const assets = await this.repository.getAssets(authUser.id, id);
// const ids = assets.map((asset) => asset.id);
// await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
// }
}
if (dto.featureFaceAssetId) {
@ -152,10 +152,10 @@ export class PersonService {
const mergeData: UpdateFacesData = { oldPersonId: mergeId, newPersonId: id };
this.logger.log(`Merging ${mergeName} into ${primaryName}`);
const assetIds = await this.repository.prepareReassignFaces(mergeData);
for (const assetId of assetIds) {
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId: mergeId } });
}
// const assetIds = await this.repository.prepareReassignFaces(mergeData);
// for (const assetId of assetIds) {
// await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId: mergeId } });
// }
await this.repository.reassignFaces(mergeData);
await this.repository.delete(mergePerson);
@ -168,7 +168,7 @@ export class PersonService {
}
// Re-index all faces in typesense for up-to-date search results
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
// await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
return results;
}

796
server/src/domain/search/search.service.spec.ts
View file

@ -1,431 +1,431 @@
import { BadRequestException } from '@nestjs/common';
import {
albumStub,
assetStub,
asyncTick,
authStub,
faceStub,
newAlbumRepositoryMock,
newAssetRepositoryMock,
newFaceRepositoryMock,
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newSearchRepositoryMock,
newSystemConfigRepositoryMock,
searchStub,
} from '@test';
import { plainToInstance } from 'class-transformer';
import { IAlbumRepository } from '../album/album.repository';
import { mapAsset } from '../asset';
import { IAssetRepository } from '../asset/asset.repository';
import { IFaceRepository } from '../facial-recognition';
import { JobName } from '../job';
import { IJobRepository } from '../job/job.repository';
import { IMachineLearningRepository } from '../smart-info';
import { ISystemConfigRepository } from '../system-config';
import { SearchDto } from './dto';
import { ISearchRepository } from './search.repository';
import { SearchService } from './search.service';
jest.useFakeTimers();
describe(SearchService.name, () => {
let sut: SearchService;
let albumMock: jest.Mocked<IAlbumRepository>;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let faceMock: jest.Mocked<IFaceRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
beforeEach(async () => {
albumMock = newAlbumRepositoryMock();
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
faceMock = newFaceRepositoryMock();
jobMock = newJobRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
searchMock = newSearchRepositoryMock();
sut = new SearchService(albumMock, assetMock, configMock, faceMock, jobMock, machineMock, searchMock);
searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false, faces: false });
delete process.env.TYPESENSE_ENABLED;
await sut.init();
});
const disableSearch = () => {
searchMock.setup.mockClear();
searchMock.checkMigrationStatus.mockClear();
jobMock.queue.mockClear();
process.env.TYPESENSE_ENABLED = 'false';
};
afterEach(() => {
sut.teardown();
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('request dto', () => {
it('should convert smartInfo.tags to a string list', () => {
const instance = plainToInstance(SearchDto, { 'smartInfo.tags': 'a,b,c' });
expect(instance['smartInfo.tags']).toEqual(['a', 'b', 'c']);
});
it('should handle empty smartInfo.tags', () => {
const instance = plainToInstance(SearchDto, {});
expect(instance['smartInfo.tags']).toBeUndefined();
});
it('should convert smartInfo.objects to a string list', () => {
const instance = plainToInstance(SearchDto, { 'smartInfo.objects': 'a,b,c' });
expect(instance['smartInfo.objects']).toEqual(['a', 'b', 'c']);
});
it('should handle empty smartInfo.objects', () => {
const instance = plainToInstance(SearchDto, {});
expect(instance['smartInfo.objects']).toBeUndefined();
});
});
describe(`init`, () => {
it('should skip when search is disabled', async () => {
disableSearch();
await sut.init();
expect(searchMock.setup).not.toHaveBeenCalled();
expect(searchMock.checkMigrationStatus).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should skip schema migration if not needed', async () => {
await sut.init();
expect(searchMock.setup).toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should do schema migration if needed', async () => {
searchMock.checkMigrationStatus.mockResolvedValue({ assets: true, albums: true, faces: true });
await sut.init();
expect(searchMock.setup).toHaveBeenCalled();
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_INDEX_ASSETS }],
[{ name: JobName.SEARCH_INDEX_ALBUMS }],
[{ name: JobName.SEARCH_INDEX_FACES }],
]);
});
});
describe('getExploreData', () => {
it('should throw bad request exception if search is disabled', async () => {
disableSearch();
await expect(sut.getExploreData(authStub.admin)).rejects.toBeInstanceOf(BadRequestException);
expect(searchMock.explore).not.toHaveBeenCalled();
});
it('should return explore data if feature flag SEARCH is set', async () => {
searchMock.explore.mockResolvedValue([{ fieldName: 'name', items: [{ value: 'image', data: assetStub.image }] }]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await expect(sut.getExploreData(authStub.admin)).resolves.toEqual([
{
fieldName: 'name',
items: [{ value: 'image', data: mapAsset(assetStub.image) }],
},
]);
expect(searchMock.explore).toHaveBeenCalledWith(authStub.admin.id);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
});
});
describe('search', () => {
// it('should throw an error is search is disabled', async () => {
// sut['enabled'] = false;
// await expect(sut.search(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException);
// expect(searchMock.searchAlbums).not.toHaveBeenCalled();
// expect(searchMock.searchAssets).not.toHaveBeenCalled();
// });
it('should search assets and albums using text search', async () => {
searchMock.searchAssets.mockResolvedValue(searchStub.withImage);
searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await expect(sut.search(authStub.admin, {})).resolves.toEqual({
albums: {
total: 0,
count: 0,
page: 1,
items: [],
facets: [],
distances: [],
},
assets: {
total: 1,
count: 1,
page: 1,
items: [mapAsset(assetStub.image)],
facets: [],
distances: [],
},
});
// expect(searchMock.searchAssets).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
expect(searchMock.searchAlbums).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
});
it('should search assets and albums using vector search', async () => {
searchMock.vectorSearch.mockResolvedValue(searchStub.emptyResults);
searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
machineMock.encodeText.mockResolvedValue([123]);
await expect(sut.search(authStub.admin, { clip: true, query: 'foo' })).resolves.toEqual({
albums: {
total: 0,
count: 0,
page: 1,
items: [],
facets: [],
distances: [],
},
assets: {
total: 0,
count: 0,
page: 1,
items: [],
facets: [],
distances: [],
},
});
expect(machineMock.encodeText).toHaveBeenCalledWith(expect.any(String), { text: 'foo' }, expect.any(Object));
expect(searchMock.vectorSearch).toHaveBeenCalledWith([123], {
userId: authStub.admin.id,
clip: true,
query: 'foo',
});
expect(searchMock.searchAlbums).toHaveBeenCalledWith('foo', {
userId: authStub.admin.id,
clip: true,
query: 'foo',
});
});
});
describe('handleIndexAssets', () => {
it('should call done, even when there are no assets', async () => {
await sut.handleIndexAssets();
expect(searchMock.importAssets).toHaveBeenCalledWith([], true);
});
it('should index all the assets', async () => {
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
await sut.handleIndexAssets();
expect(searchMock.importAssets.mock.calls).toEqual([
[[assetStub.image], false],
[[], true],
]);
});
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexAssets();
expect(searchMock.importAssets).not.toHaveBeenCalled();
expect(searchMock.importAlbums).not.toHaveBeenCalled();
});
});
describe('handleIndexAsset', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexAsset({ ids: [assetStub.image.id] });
});
it('should index the asset', () => {
sut.handleIndexAsset({ ids: [assetStub.image.id] });
});
});
describe('handleIndexAlbums', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexAlbums();
});
it('should index all the albums', async () => {
albumMock.getAll.mockResolvedValue([albumStub.empty]);
await sut.handleIndexAlbums();
expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], true);
});
});
describe('handleIndexAlbum', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
});
it('should index the album', () => {
sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
});
});
describe('handleRemoveAlbum', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveAlbum({ ids: ['album1'] });
});
it('should remove the album', () => {
sut.handleRemoveAlbum({ ids: ['album1'] });
});
});
describe('handleRemoveAsset', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveAsset({ ids: ['asset1'] });
});
it('should remove the asset', () => {
sut.handleRemoveAsset({ ids: ['asset1'] });
});
});
describe('handleIndexFaces', () => {
it('should call done, even when there are no faces', async () => {
faceMock.getAll.mockResolvedValue([]);
// import { BadRequestException } from '@nestjs/common';
// import {
// albumStub,
// assetStub,
// asyncTick,
// authStub,
// faceStub,
// newAlbumRepositoryMock,
// newAssetRepositoryMock,
// newFaceRepositoryMock,
// newJobRepositoryMock,
// newMachineLearningRepositoryMock,
// newSearchRepositoryMock,
// newSystemConfigRepositoryMock,
// searchStub,
// } from '@test';
// import { plainToInstance } from 'class-transformer';
// import { IAlbumRepository } from '../album/album.repository';
// import { mapAsset } from '../asset';
// import { IAssetRepository } from '../asset/asset.repository';
// import { IFaceRepository } from '../facial-recognition';
// // import { JobName } from '../job';
// import { IJobRepository } from '../job/job.repository';
// import { IMachineLearningRepository } from '../smart-info';
// import { ISystemConfigRepository } from '../system-config';
// import { SearchDto } from './dto';
// import { ISearchRepository } from './search.repository';
// import { SearchService } from './search.service';
// jest.useFakeTimers();
// describe(SearchService.name, () => {
// let sut: SearchService;
// let albumMock: jest.Mocked<IAlbumRepository>;
// let assetMock: jest.Mocked<IAssetRepository>;
// let configMock: jest.Mocked<ISystemConfigRepository>;
// let faceMock: jest.Mocked<IFaceRepository>;
// let jobMock: jest.Mocked<IJobRepository>;
// let machineMock: jest.Mocked<IMachineLearningRepository>;
// let searchMock: jest.Mocked<ISearchRepository>;
// beforeEach(async () => {
// albumMock = newAlbumRepositoryMock();
// assetMock = newAssetRepositoryMock();
// configMock = newSystemConfigRepositoryMock();
// faceMock = newFaceRepositoryMock();
// jobMock = newJobRepositoryMock();
// machineMock = newMachineLearningRepositoryMock();
// searchMock = newSearchRepositoryMock();
// sut = new SearchService(albumMock, assetMock, configMock, faceMock, jobMock, machineMock, searchMock);
// searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false, faces: false });
// delete process.env.TYPESENSE_ENABLED;
// await sut.init();
// });
// const disableSearch = () => {
// searchMock.setup.mockClear();
// searchMock.checkMigrationStatus.mockClear();
// jobMock.queue.mockClear();
// process.env.TYPESENSE_ENABLED = 'false';
// };
// afterEach(() => {
// sut.teardown();
// });
// it('should work', () => {
// expect(sut).toBeDefined();
// });
// describe('request dto', () => {
// it('should convert smartInfo.tags to a string list', () => {
// const instance = plainToInstance(SearchDto, { 'smartInfo.tags': 'a,b,c' });
// expect(instance['smartInfo.tags']).toEqual(['a', 'b', 'c']);
// });
// it('should handle empty smartInfo.tags', () => {
// const instance = plainToInstance(SearchDto, {});
// expect(instance['smartInfo.tags']).toBeUndefined();
// });
// it('should convert smartInfo.objects to a string list', () => {
// const instance = plainToInstance(SearchDto, { 'smartInfo.objects': 'a,b,c' });
// expect(instance['smartInfo.objects']).toEqual(['a', 'b', 'c']);
// });
// it('should handle empty smartInfo.objects', () => {
// const instance = plainToInstance(SearchDto, {});
// expect(instance['smartInfo.objects']).toBeUndefined();
// });
// });
// describe(`init`, () => {
// it('should skip when search is disabled', async () => {
// disableSearch();
// await sut.init();
// expect(searchMock.setup).not.toHaveBeenCalled();
// expect(searchMock.checkMigrationStatus).not.toHaveBeenCalled();
// expect(jobMock.queue).not.toHaveBeenCalled();
// });
// it('should skip schema migration if not needed', async () => {
// await sut.init();
// expect(searchMock.setup).toHaveBeenCalled();
// expect(jobMock.queue).not.toHaveBeenCalled();
// });
// it('should do schema migration if needed', async () => {
// searchMock.checkMigrationStatus.mockResolvedValue({ assets: true, albums: true, faces: true });
// await sut.init();
// expect(searchMock.setup).toHaveBeenCalled();
// expect(jobMock.queue.mock.calls).toEqual([
// [{ name: JobName.SEARCH_INDEX_ASSETS }],
// [{ name: JobName.SEARCH_INDEX_ALBUMS }],
// [{ name: JobName.SEARCH_INDEX_FACES }],
// ]);
// });
// });
// describe('getExploreData', () => {
// it('should throw bad request exception if search is disabled', async () => {
// disableSearch();
// await expect(sut.getExploreData(authStub.admin)).rejects.toBeInstanceOf(BadRequestException);
// expect(searchMock.explore).not.toHaveBeenCalled();
// });
// it('should return explore data if feature flag SEARCH is set', async () => {
// searchMock.explore.mockResolvedValue([{ fieldName: 'name', items: [{ value: 'image', data: assetStub.image }] }]);
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
// await expect(sut.getExploreData(authStub.admin)).resolves.toEqual([
// {
// fieldName: 'name',
// items: [{ value: 'image', data: mapAsset(assetStub.image) }],
// },
// ]);
// expect(searchMock.explore).toHaveBeenCalledWith(authStub.admin.id);
// expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
// });
// });
// describe('search', () => {
// // it('should throw an error is search is disabled', async () => {
// // sut['enabled'] = false;
// // await expect(sut.search(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException);
// // expect(searchMock.searchAlbums).not.toHaveBeenCalled();
// // expect(searchMock.searchAssets).not.toHaveBeenCalled();
// // });
// it('should search assets and albums using text search', async () => {
// searchMock.searchAssets.mockResolvedValue(searchStub.withImage);
// searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
// await expect(sut.search(authStub.admin, {})).resolves.toEqual({
// albums: {
// total: 0,
// count: 0,
// page: 1,
// items: [],
// facets: [],
// distances: [],
// },
// assets: {
// total: 1,
// count: 1,
// page: 1,
// items: [mapAsset(assetStub.image)],
// facets: [],
// distances: [],
// },
// });
// expect(searchMock.searchAssets).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
// expect(searchMock.searchAlbums).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
// });
// it('should search assets and albums using vector search', async () => {
// searchMock.vectorSearch.mockResolvedValue(searchStub.emptyResults);
// searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
// machineMock.encodeText.mockResolvedValue([123]);
// await expect(sut.search(authStub.admin, { clip: true, query: 'foo' })).resolves.toEqual({
// albums: {
// total: 0,
// count: 0,
// page: 1,
// items: [],
// facets: [],
// distances: [],
// },
// assets: {
// total: 0,
// count: 0,
// page: 1,
// items: [],
// facets: [],
// distances: [],
// },
// });
// expect(machineMock.encodeText).toHaveBeenCalledWith(expect.any(String), { text: 'foo' }, expect.any(Object));
// expect(searchMock.vectorSearch).toHaveBeenCalledWith([123], {
// userId: authStub.admin.id,
// clip: true,
// query: 'foo',
// });
// expect(searchMock.searchAlbums).toHaveBeenCalledWith('foo', {
// userId: authStub.admin.id,
// clip: true,
// query: 'foo',
// });
// });
// });
// describe('handleIndexAssets', () => {
// it('should call done, even when there are no assets', async () => {
// await sut.handleIndexAssets();
// expect(searchMock.importAssets).toHaveBeenCalledWith([], true);
// });
// it('should index all the assets', async () => {
// assetMock.getAll.mockResolvedValue({
// items: [assetStub.image],
// hasNextPage: false,
// });
// await sut.handleIndexAssets();
// expect(searchMock.importAssets.mock.calls).toEqual([
// [[assetStub.image], false],
// [[], true],
// ]);
// });
// it('should skip if search is disabled', async () => {
// sut['enabled'] = false;
// await sut.handleIndexAssets();
// expect(searchMock.importAssets).not.toHaveBeenCalled();
// expect(searchMock.importAlbums).not.toHaveBeenCalled();
// });
// });
// describe('handleIndexAsset', () => {
// it('should skip if search is disabled', () => {
// sut['enabled'] = false;
// sut.handleIndexAsset({ ids: [assetStub.image.id] });
// });
// it('should index the asset', () => {
// sut.handleIndexAsset({ ids: [assetStub.image.id] });
// });
// });
// describe('handleIndexAlbums', () => {
// it('should skip if search is disabled', () => {
// sut['enabled'] = false;
// sut.handleIndexAlbums();
// });
// it('should index all the albums', async () => {
// albumMock.getAll.mockResolvedValue([albumStub.empty]);
// await sut.handleIndexAlbums();
// expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], true);
// });
// });
// describe('handleIndexAlbum', () => {
// it('should skip if search is disabled', () => {
// sut['enabled'] = false;
// sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
// });
// it('should index the album', () => {
// sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
// });
// });
// describe('handleRemoveAlbum', () => {
// it('should skip if search is disabled', () => {
// sut['enabled'] = false;
// sut.handleRemoveAlbum({ ids: ['album1'] });
// });
// it('should remove the album', () => {
// sut.handleRemoveAlbum({ ids: ['album1'] });
// });
// });
// describe('handleRemoveAsset', () => {
// it('should skip if search is disabled', () => {
// sut['enabled'] = false;
// sut.handleRemoveAsset({ ids: ['asset1'] });
// });
// it('should remove the asset', () => {
// sut.handleRemoveAsset({ ids: ['asset1'] });
// });
// });
// describe('handleIndexFaces', () => {
// it('should call done, even when there are no faces', async () => {
// faceMock.getAll.mockResolvedValue([]);
await sut.handleIndexFaces();
// await sut.handleIndexFaces();
expect(searchMock.importFaces).toHaveBeenCalledWith([], true);
});
// expect(searchMock.importFaces).toHaveBeenCalledWith([], true);
// });
it('should index all the faces', async () => {
faceMock.getAll.mockResolvedValue([faceStub.face1]);
// it('should index all the faces', async () => {
// faceMock.getAll.mockResolvedValue([faceStub.face1]);
await sut.handleIndexFaces();
// await sut.handleIndexFaces();
expect(searchMock.importFaces.mock.calls).toEqual([
[
[
{
id: 'asset-id|person-1',
ownerId: 'user-id',
assetId: 'asset-id',
personId: 'person-1',
embedding: [1, 2, 3, 4],
},
],
false,
],
[[], true],
]);
});
// expect(searchMock.importFaces.mock.calls).toEqual([
// [
// [
// {
// id: 'asset-id|person-1',
// ownerId: 'user-id',
// assetId: 'asset-id',
// personId: 'person-1',
// embedding: [1, 2, 3, 4],
// },
// ],
// false,
// ],
// [[], true],
// ]);
// });
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
// it('should skip if search is disabled', async () => {
// sut['enabled'] = false;
await sut.handleIndexFaces();
// await sut.handleIndexFaces();
expect(searchMock.importFaces).not.toHaveBeenCalled();
});
});
// expect(searchMock.importFaces).not.toHaveBeenCalled();
// });
// });
describe('handleIndexAsset', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
// describe('handleIndexAsset', () => {
// it('should skip if search is disabled', () => {
// sut['enabled'] = false;
// sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
expect(searchMock.importFaces).not.toHaveBeenCalled();
expect(faceMock.getByIds).not.toHaveBeenCalled();
});
// expect(searchMock.importFaces).not.toHaveBeenCalled();
// expect(faceMock.getByIds).not.toHaveBeenCalled();
// });
it('should index the face', () => {
faceMock.getByIds.mockResolvedValue([faceStub.face1]);
// it('should index the face', () => {
// faceMock.getByIds.mockResolvedValue([faceStub.face1]);
sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
// sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
expect(faceMock.getByIds).toHaveBeenCalledWith([{ assetId: 'asset-1', personId: 'person-1' }]);
});
});
// expect(faceMock.getByIds).toHaveBeenCalledWith([{ assetId: 'asset-1', personId: 'person-1' }]);
// });
// });
describe('handleRemoveFace', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
});
// describe('handleRemoveFace', () => {
// it('should skip if search is disabled', () => {
// sut['enabled'] = false;
// sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
// });
it('should remove the face', () => {
sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
});
});
// it('should remove the face', () => {
// sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
// });
// });
describe('flush', () => {
it('should flush queued album updates', async () => {
albumMock.getByIds.mockResolvedValue([albumStub.empty]);
// describe('flush', () => {
// it('should flush queued album updates', async () => {
// albumMock.getByIds.mockResolvedValue([albumStub.empty]);
sut.handleIndexAlbum({ ids: ['album1'] });
// sut.handleIndexAlbum({ ids: ['album1'] });
jest.runOnlyPendingTimers();
// jest.runOnlyPendingTimers();
await asyncTick(4);
// await asyncTick(4);
expect(albumMock.getByIds).toHaveBeenCalledWith(['album1']);
expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], false);
});
// expect(albumMock.getByIds).toHaveBeenCalledWith(['album1']);
// expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], false);
// });
it('should flush queued album deletes', async () => {
sut.handleRemoveAlbum({ ids: ['album1'] });
// it('should flush queued album deletes', async () => {
// sut.handleRemoveAlbum({ ids: ['album1'] });
jest.runOnlyPendingTimers();
// jest.runOnlyPendingTimers();
await asyncTick(4);
// await asyncTick(4);
expect(searchMock.deleteAlbums).toHaveBeenCalledWith(['album1']);
});
// expect(searchMock.deleteAlbums).toHaveBeenCalledWith(['album1']);
// });
it('should flush queued asset updates', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
// it('should flush queued asset updates', async () => {
// assetMock.getByIds.mockResolvedValue([assetStub.image]);
sut.handleIndexAsset({ ids: ['asset1'] });
// sut.handleIndexAsset({ ids: ['asset1'] });
jest.runOnlyPendingTimers();
// jest.runOnlyPendingTimers();
await asyncTick(4);
// await asyncTick(4);
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset1']);
expect(searchMock.importAssets).toHaveBeenCalledWith([assetStub.image], false);
});
// expect(assetMock.getByIds).toHaveBeenCalledWith(['asset1']);
// expect(searchMock.importAssets).toHaveBeenCalledWith([assetStub.image], false);
// });
it('should flush queued asset deletes', async () => {
sut.handleRemoveAsset({ ids: ['asset1'] });
// it('should flush queued asset deletes', async () => {
// sut.handleRemoveAsset({ ids: ['asset1'] });
jest.runOnlyPendingTimers();
// jest.runOnlyPendingTimers();
await asyncTick(4);
// await asyncTick(4);
expect(searchMock.deleteAssets).toHaveBeenCalledWith(['asset1']);
});
});
});
// expect(searchMock.deleteAssets).toHaveBeenCalledWith(['asset1']);
// });
// });
// });

332
server/src/domain/search/search.service.ts
View file

@ -29,91 +29,19 @@ interface SyncQueue {
@Injectable()
export class SearchService {
private logger = new Logger(SearchService.name);
private enabled = false;
private timer: NodeJS.Timeout | null = null;
private configCore: SystemConfigCore;
private albumQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
private assetQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
private faceQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
constructor(
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IFaceRepository) private faceRepository: IFaceRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
) {
this.configCore = new SystemConfigCore(configRepository);
}
teardown() {
if (this.timer) {
clearInterval(this.timer);
this.timer = null;
}
}
async init() {
this.enabled = await this.configCore.hasFeature(FeatureFlag.SEARCH);
if (!this.enabled) {
return;
}
this.logger.log('Running bootstrap');
await this.searchRepository.setup();
const migrationStatus = await this.searchRepository.checkMigrationStatus();
if (migrationStatus[SearchCollection.ASSETS]) {
this.logger.debug('Queueing job to re-index all assets');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSETS });
}
if (migrationStatus[SearchCollection.ALBUMS]) {
this.logger.debug('Queueing job to re-index all albums');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUMS });
}
if (migrationStatus[SearchCollection.FACES]) {
this.logger.debug('Queueing job to re-index all faces');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
}
this.timer = setInterval(() => this.flush(), 5_000);
}
async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
await this.configCore.requireFeature(FeatureFlag.SEARCH);
const results = await this.searchRepository.explore(authUser.id);
const lookup = await this.getLookupMap(
results.reduce(
(ids: string[], result: SearchExploreItem<AssetEntity>) => [
...ids,
...result.items.map((item) => item.data.id),
],
[],
),
);
return results.map(({ fieldName, items }) => ({
fieldName,
items: items
.map(({ value, data }) => ({ value, data: lookup[data.id] }))
.filter(({ data }) => !!data)
.map(({ value, data }) => ({ value, data: mapAsset(data) })),
}));
return []
}
async search(authUser: AuthUserDto, dto: SearchDto): Promise<SearchResponseDto> {
@ -123,259 +51,37 @@ export class SearchService {
const query = dto.q || dto.query || '*';
const hasClip = machineLearning.enabled && machineLearning.clip.enabled;
const strategy = dto.clip && hasClip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
const filters = { userId: authUser.id, ...dto };
let assets: SearchResult<AssetEntity>;
let assets: AssetEntity[];
let ids;
switch (strategy) {
case SearchStrategy.CLIP:
const {
machineLearning: { clip },
} = await this.configCore.getConfig();
const embedding = await this.machineLearning.encodeText(machineLearning.url, { text: query }, clip);
assets = await this.searchRepository.vectorSearch(embedding, filters);
ids = await this.machineLearning.encodeText(machineLearning.url, { text: query }, { ...clip, index_name: `${authUser.id}-${JobName.ENCODE_CLIP}`, k: 100 }) as string[];
assets = await this.assetRepository.getByIds(ids)
break;
case SearchStrategy.TEXT:
default:
assets = await this.searchRepository.searchAssets(query, filters);
break;
throw new Error('Not implemented');
}
const albums = await this.searchRepository.searchAlbums(query, filters);
const lookup = await this.getLookupMap(assets.items.map((asset) => asset.id));
return {
albums: { ...albums, items: albums.items.map(mapAlbumWithAssets) },
assets: {
...assets,
items: assets.items
.map((item) => lookup[item.id])
.filter((item) => !!item)
.map(mapAsset),
albums: {
total: 0,
count: 0,
items: [],
facets: [],
},
assets: {
total: assets.length,
count: assets.length,
items: assets
.filter((asset) => !!asset)
.map(mapAsset),
facets: []
}
};
}
async handleIndexAlbums() {
if (!this.enabled) {
return false;
}
const albums = this.patchAlbums(await this.albumRepository.getAll());
this.logger.log(`Indexing ${albums.length} albums`);
await this.searchRepository.importAlbums(albums, true);
return true;
}
async handleIndexAssets() {
if (!this.enabled) {
return false;
}
// TODO: do this in batches based on searchIndexVersion
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getAll(pagination, { isVisible: true }),
);
for await (const assets of assetPagination) {
this.logger.debug(`Indexing ${assets.length} assets`);
const patchedAssets = this.patchAssets(assets);
await this.searchRepository.importAssets(patchedAssets, false);
}
await this.searchRepository.importAssets([], true);
this.logger.debug('Finished re-indexing all assets');
return false;
}
async handleIndexFaces() {
if (!this.enabled) {
return false;
}
await this.searchRepository.deleteAllFaces();
// TODO: do this in batches based on searchIndexVersion
const faces = this.patchFaces(await this.faceRepository.getAll());
this.logger.log(`Indexing ${faces.length} faces`);
const chunkSize = 1000;
for (let i = 0; i < faces.length; i += chunkSize) {
await this.searchRepository.importFaces(faces.slice(i, i + chunkSize), false);
}
await this.searchRepository.importFaces([], true);
this.logger.debug('Finished re-indexing all faces');
return true;
}
handleIndexAlbum({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.albumQueue.upsert.add(id);
}
return true;
}
handleIndexAsset({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.assetQueue.upsert.add(id);
}
return true;
}
async handleIndexFace({ assetId, personId }: IAssetFaceJob) {
if (!this.enabled) {
return false;
}
// immediately push to typesense
await this.searchRepository.importFaces(await this.idsToFaces([{ assetId, personId }]), false);
return true;
}
handleRemoveAlbum({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.albumQueue.delete.add(id);
}
return true;
}
handleRemoveAsset({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.assetQueue.delete.add(id);
}
return true;
}
handleRemoveFace({ assetId, personId }: IAssetFaceJob) {
if (!this.enabled) {
return false;
}
this.faceQueue.delete.add(this.asKey({ assetId, personId }));
return true;
}
private async flush() {
if (this.albumQueue.upsert.size > 0) {
const ids = [...this.albumQueue.upsert.keys()];
const items = await this.idsToAlbums(ids);
this.logger.debug(`Flushing ${items.length} album upserts`);
await this.searchRepository.importAlbums(items, false);
this.albumQueue.upsert.clear();
}
if (this.albumQueue.delete.size > 0) {
const ids = [...this.albumQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} album deletes`);
await this.searchRepository.deleteAlbums(ids);
this.albumQueue.delete.clear();
}
if (this.assetQueue.upsert.size > 0) {
const ids = [...this.assetQueue.upsert.keys()];
const items = await this.idsToAssets(ids);
this.logger.debug(`Flushing ${items.length} asset upserts`);
await this.searchRepository.importAssets(items, false);
this.assetQueue.upsert.clear();
}
if (this.assetQueue.delete.size > 0) {
const ids = [...this.assetQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} asset deletes`);
await this.searchRepository.deleteAssets(ids);
this.assetQueue.delete.clear();
}
if (this.faceQueue.upsert.size > 0) {
const ids = [...this.faceQueue.upsert.keys()].map((key) => this.asParts(key));
const items = await this.idsToFaces(ids);
this.logger.debug(`Flushing ${items.length} face upserts`);
await this.searchRepository.importFaces(items, false);
this.faceQueue.upsert.clear();
}
if (this.faceQueue.delete.size > 0) {
const ids = [...this.faceQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} face deletes`);
await this.searchRepository.deleteFaces(ids);
this.faceQueue.delete.clear();
}
}
private async idsToAlbums(ids: string[]): Promise<AlbumEntity[]> {
const entities = await this.albumRepository.getByIds(ids);
return this.patchAlbums(entities);
}
private async idsToAssets(ids: string[]): Promise<AssetEntity[]> {
const entities = await this.assetRepository.getByIds(ids);
return this.patchAssets(entities.filter((entity) => entity.isVisible));
}
private async idsToFaces(ids: AssetFaceId[]): Promise<OwnedFaceEntity[]> {
return this.patchFaces(await this.faceRepository.getByIds(ids));
}
private patchAssets(assets: AssetEntity[]): AssetEntity[] {
return assets;
}
private patchAlbums(albums: AlbumEntity[]): AlbumEntity[] {
return albums.map((entity) => ({ ...entity, assets: [] }));
}
private patchFaces(faces: AssetFaceEntity[]): OwnedFaceEntity[] {
return faces.map((face) => ({
id: this.asKey(face),
ownerId: face.asset.ownerId,
assetId: face.assetId,
personId: face.personId,
embedding: face.embedding,
}));
}
private asKey(face: AssetFaceId): string {
return `${face.assetId}|${face.personId}`;
}
private asParts(key: string): AssetFaceId {
const [assetId, personId] = key.split('|');
return { assetId, personId };
}
private async getLookupMap(assetIds: string[]) {
const assets = await this.assetRepository.getByIds(assetIds);
const lookup: Record<string, AssetEntity> = {};
for (const asset of assets) {
lookup[asset.id] = asset;
}
return lookup;
}
}

17
server/src/domain/smart-info/dto/model-config.dto.ts
View file

@ -1,6 +1,6 @@
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsBoolean, IsEnum, IsNotEmpty, IsNumber, IsString, Max, Min } from 'class-validator';
import { IsBoolean, IsEnum, IsInt, IsNotEmpty, IsNumber, IsString, Max, Min } from 'class-validator';
import { Optional } from '../../domain.util';
import { CLIPMode, ModelType } from '../machine-learning.interface';
@ -16,6 +16,21 @@ export class ModelConfig {
@Optional()
@ApiProperty({ enumName: 'ModelType', enum: ModelType })
modelType?: ModelType;
@IsString()
@IsNotEmpty()
@Optional()
index_name?: string;
@IsString()
@IsNotEmpty()
@Optional()
embedding_id?: string;
@IsInt()
@Min(1)
@Optional()
k?: number;
}
export class ClassificationConfig extends ModelConfig {

6
server/src/domain/smart-info/machine-learning.interface.ts
View file

@ -38,7 +38,7 @@ export enum CLIPMode {
export interface IMachineLearningRepository {
classifyImage(url: string, input: VisionModelInput, config: ClassificationConfig): Promise<string[]>;
encodeImage(url: string, input: VisionModelInput, config: CLIPConfig): Promise<number[]>;
encodeText(url: string, input: TextModelInput, config: CLIPConfig): Promise<number[]>;
detectFaces(url: string, input: VisionModelInput, config: RecognitionConfig): Promise<DetectFaceResult[]>;
encodeImage(url: string, input: VisionModelInput, config: CLIPConfig): Promise<number[] | string[]>;
encodeText(url: string, input: TextModelInput, config: CLIPConfig): Promise<number[] | string[]>;
detectFaces(url: string, input: VisionModelInput, config: RecognitionConfig): Promise<DetectFaceResult[] | string[]>;
}

4
server/src/domain/smart-info/smart-info.service.ts
View file

@ -97,8 +97,8 @@ export class SmartInfoService {
const clipEmbedding = await this.machineLearning.encodeImage(
machineLearning.url,
{ imagePath: asset.resizePath },
machineLearning.clip,
);
{ ...machineLearning.clip, index_name: `${asset.ownerId}-${JobName.ENCODE_CLIP}`, embedding_id: asset.id },
) as number[];
await this.repository.upsert({ assetId: asset.id, clipEmbedding: clipEmbedding });

16
server/src/immich/api-v1/asset/asset.service.ts
View file

@ -264,16 +264,16 @@ export class AssetService {
}
try {
if (asset.faces) {
await Promise.all(
asset.faces.map(({ assetId, personId }) =>
this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
),
);
}
// if (asset.faces) {
// await Promise.all(
// asset.faces.map(({ assetId, personId }) =>
// this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
// ),
// );
// }
await this._assetRepository.remove(asset);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [id] } });
// await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [id] } });
result.push({ id, status: DeleteAssetStatusEnum.SUCCESS });

8
server/src/immich/app.module.ts
View file

@ -66,14 +66,10 @@ import {
FileUploadInterceptor,
],
})
export class AppModule implements OnModuleInit, OnModuleDestroy {
constructor(private appService: AppService) {}
export class AppModule implements OnModuleInit {
constructor(private appService: AppService) { }
async onModuleInit() {
await this.appService.init();
}
onModuleDestroy() {
this.appService.destroy();
}
}

7
server/src/immich/app.service.ts
View file

@ -11,7 +11,7 @@ export class AppService {
private searchService: SearchService,
private storageService: StorageService,
private serverService: ServerInfoService,
) {}
) { }
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async onNightlyJob() {
@ -20,11 +20,6 @@ export class AppService {
async init() {
this.storageService.init();
await this.searchService.init();
this.logger.log(`Feature Flags: ${JSON.stringify(await this.serverService.getFeatures(), null, 2)}`);
}
async destroy() {
this.searchService.teardown();
}
}

18
server/src/infra/repositories/machine-learning.repository.ts
View file

@ -17,7 +17,7 @@ import { readFile } from 'fs/promises';
export class MachineLearningRepository implements IMachineLearningRepository {
private async post<T>(url: string, input: TextModelInput | VisionModelInput, config: ModelConfig): Promise<T> {
const formData = await this.getFormData(input, config);
const res = await fetch(`${url}/predict`, { method: 'POST', body: formData });
const res = await fetch(`${url}/pipeline`, { method: 'POST', body: formData });
if (res.status >= 400) {
throw new Error(
`Request ${config.modelType ? `for ${config.modelType.replace('-', ' ')} ` : ''}` +
@ -31,11 +31,11 @@ export class MachineLearningRepository implements IMachineLearningRepository {
return this.post<string[]>(url, input, { ...config, modelType: ModelType.IMAGE_CLASSIFICATION });
}
detectFaces(url: string, input: VisionModelInput, config: RecognitionConfig): Promise<DetectFaceResult[]> {
detectFaces(url: string, input: VisionModelInput, config: RecognitionConfig): Promise<DetectFaceResult[] | string[]> {
return this.post<DetectFaceResult[]>(url, input, { ...config, modelType: ModelType.FACIAL_RECOGNITION });
}
encodeImage(url: string, input: VisionModelInput, config: CLIPConfig): Promise<number[]> {
encodeImage(url: string, input: VisionModelInput, config: CLIPConfig): Promise<number[] | string[]> {
return this.post<number[]>(url, input, {
...config,
modelType: ModelType.CLIP,
@ -43,7 +43,7 @@ export class MachineLearningRepository implements IMachineLearningRepository {
} as CLIPConfig);
}
encodeText(url: string, input: TextModelInput, config: CLIPConfig): Promise<number[]> {
encodeText(url: string, input: TextModelInput, config: CLIPConfig): Promise<number[] | string[]> {
return this.post<number[]>(url, input, { ...config, modelType: ModelType.CLIP, mode: CLIPMode.TEXT } as CLIPConfig);
}
@ -69,6 +69,16 @@ export class MachineLearningRepository implements IMachineLearningRepository {
throw new Error('Invalid input');
}
if (config.index_name) {
formData.append('index_name', config.index_name);
}
if (config.embedding_id) {
formData.append('embedding_id', config.embedding_id);
}
if (config.k) {
formData.append('k', config.k.toString());
}
return formData;
}
}

12
server/src/microservices/app.service.ts
View file

@ -37,7 +37,7 @@ export class AppService {
private systemConfigService: SystemConfigService,
private userService: UserService,
private auditService: AuditService,
) {}
) { }
async init() {
await this.jobService.registerHandlers({
@ -49,15 +49,6 @@ export class AppService {
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
@ -90,6 +81,5 @@ export class AppService {
});
await this.metadataProcessor.init();
await this.searchService.init();
}
}