feat(server): return asset checksum (#2582)

* feat: return asset checksum

* chore: generate open api

* chore: coverage

* feat(server): support base64 hashes in bulk upload check:

* chore: generate open api
This commit is contained in:
Jason Rasmussen 2023-05-27 21:56:17 -04:00 committed by GitHub
parent 7f0ad8e2d2
commit bca4626708
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 111 additions and 103 deletions

View file

@ -9,7 +9,7 @@ import 'package:openapi/api.dart';
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**id** | **String** | |
**checksum** | **String** | |
**checksum** | **String** | base64 or hex encoded sha1 hash |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

View file

@ -15,7 +15,7 @@ Name | Type | Description | Notes
**deviceId** | **String** | |
**originalPath** | **String** | |
**originalFileName** | **String** | |
**resizePath** | **String** | |
**resized** | **bool** | |
**fileCreatedAt** | **String** | |
**fileModifiedAt** | **String** | |
**updatedAt** | **String** | |
@ -23,13 +23,12 @@ Name | Type | Description | Notes
**isArchived** | **bool** | |
**mimeType** | **String** | |
**duration** | **String** | |
**webpPath** | **String** | |
**encodedVideoPath** | **String** | | [optional]
**exifInfo** | [**ExifResponseDto**](ExifResponseDto.md) | | [optional]
**smartInfo** | [**SmartInfoResponseDto**](SmartInfoResponseDto.md) | | [optional]
**livePhotoVideoId** | **String** | | [optional]
**tags** | [**List<TagResponseDto>**](TagResponseDto.md) | | [optional] [default to const []]
**people** | [**List<PersonResponseDto>**](PersonResponseDto.md) | | [optional] [default to const []]
**checksum** | **String** | base64 encoded sha1 hash |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

View file

@ -19,6 +19,7 @@ class AssetBulkUploadCheckItem {
String id;
/// base64 or hex encoded sha1 hash
String checksum;
@override

View file

@ -20,7 +20,7 @@ class AssetResponseDto {
required this.deviceId,
required this.originalPath,
required this.originalFileName,
required this.resizePath,
required this.resized,
required this.fileCreatedAt,
required this.fileModifiedAt,
required this.updatedAt,
@ -28,13 +28,12 @@ class AssetResponseDto {
required this.isArchived,
required this.mimeType,
required this.duration,
required this.webpPath,
this.encodedVideoPath,
this.exifInfo,
this.smartInfo,
this.livePhotoVideoId,
this.tags = const [],
this.people = const [],
required this.checksum,
});
AssetTypeEnum type;
@ -51,7 +50,7 @@ class AssetResponseDto {
String originalFileName;
String? resizePath;
bool resized;
String fileCreatedAt;
@ -67,10 +66,6 @@ class AssetResponseDto {
String duration;
String? webpPath;
String? encodedVideoPath;
///
/// Please note: This property should have been non-nullable! Since the specification file
/// does not include a default value (using the "default:" property), however, the generated
@ -93,6 +88,9 @@ class AssetResponseDto {
List<PersonResponseDto> people;
/// base64 encoded sha1 hash
String checksum;
@override
bool operator ==(Object other) => identical(this, other) || other is AssetResponseDto &&
other.type == type &&
@ -102,7 +100,7 @@ class AssetResponseDto {
other.deviceId == deviceId &&
other.originalPath == originalPath &&
other.originalFileName == originalFileName &&
other.resizePath == resizePath &&
other.resized == resized &&
other.fileCreatedAt == fileCreatedAt &&
other.fileModifiedAt == fileModifiedAt &&
other.updatedAt == updatedAt &&
@ -110,13 +108,12 @@ class AssetResponseDto {
other.isArchived == isArchived &&
other.mimeType == mimeType &&
other.duration == duration &&
other.webpPath == webpPath &&
other.encodedVideoPath == encodedVideoPath &&
other.exifInfo == exifInfo &&
other.smartInfo == smartInfo &&
other.livePhotoVideoId == livePhotoVideoId &&
other.tags == tags &&
other.people == people;
other.people == people &&
other.checksum == checksum;
@override
int get hashCode =>
@ -128,7 +125,7 @@ class AssetResponseDto {
(deviceId.hashCode) +
(originalPath.hashCode) +
(originalFileName.hashCode) +
(resizePath == null ? 0 : resizePath!.hashCode) +
(resized.hashCode) +
(fileCreatedAt.hashCode) +
(fileModifiedAt.hashCode) +
(updatedAt.hashCode) +
@ -136,16 +133,15 @@ class AssetResponseDto {
(isArchived.hashCode) +
(mimeType == null ? 0 : mimeType!.hashCode) +
(duration.hashCode) +
(webpPath == null ? 0 : webpPath!.hashCode) +
(encodedVideoPath == null ? 0 : encodedVideoPath!.hashCode) +
(exifInfo == null ? 0 : exifInfo!.hashCode) +
(smartInfo == null ? 0 : smartInfo!.hashCode) +
(livePhotoVideoId == null ? 0 : livePhotoVideoId!.hashCode) +
(tags.hashCode) +
(people.hashCode);
(people.hashCode) +
(checksum.hashCode);
@override
String toString() => 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, originalFileName=$originalFileName, resizePath=$resizePath, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, isArchived=$isArchived, mimeType=$mimeType, duration=$duration, webpPath=$webpPath, encodedVideoPath=$encodedVideoPath, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags, people=$people]';
String toString() => 'AssetResponseDto[type=$type, id=$id, deviceAssetId=$deviceAssetId, ownerId=$ownerId, deviceId=$deviceId, originalPath=$originalPath, originalFileName=$originalFileName, resized=$resized, fileCreatedAt=$fileCreatedAt, fileModifiedAt=$fileModifiedAt, updatedAt=$updatedAt, isFavorite=$isFavorite, isArchived=$isArchived, mimeType=$mimeType, duration=$duration, exifInfo=$exifInfo, smartInfo=$smartInfo, livePhotoVideoId=$livePhotoVideoId, tags=$tags, people=$people, checksum=$checksum]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@ -156,11 +152,7 @@ class AssetResponseDto {
json[r'deviceId'] = this.deviceId;
json[r'originalPath'] = this.originalPath;
json[r'originalFileName'] = this.originalFileName;
if (this.resizePath != null) {
json[r'resizePath'] = this.resizePath;
} else {
// json[r'resizePath'] = null;
}
json[r'resized'] = this.resized;
json[r'fileCreatedAt'] = this.fileCreatedAt;
json[r'fileModifiedAt'] = this.fileModifiedAt;
json[r'updatedAt'] = this.updatedAt;
@ -172,16 +164,6 @@ class AssetResponseDto {
// json[r'mimeType'] = null;
}
json[r'duration'] = this.duration;
if (this.webpPath != null) {
json[r'webpPath'] = this.webpPath;
} else {
// json[r'webpPath'] = null;
}
if (this.encodedVideoPath != null) {
json[r'encodedVideoPath'] = this.encodedVideoPath;
} else {
// json[r'encodedVideoPath'] = null;
}
if (this.exifInfo != null) {
json[r'exifInfo'] = this.exifInfo;
} else {
@ -199,6 +181,7 @@ class AssetResponseDto {
}
json[r'tags'] = this.tags;
json[r'people'] = this.people;
json[r'checksum'] = this.checksum;
return json;
}
@ -228,7 +211,7 @@ class AssetResponseDto {
deviceId: mapValueOfType<String>(json, r'deviceId')!,
originalPath: mapValueOfType<String>(json, r'originalPath')!,
originalFileName: mapValueOfType<String>(json, r'originalFileName')!,
resizePath: mapValueOfType<String>(json, r'resizePath'),
resized: mapValueOfType<bool>(json, r'resized')!,
fileCreatedAt: mapValueOfType<String>(json, r'fileCreatedAt')!,
fileModifiedAt: mapValueOfType<String>(json, r'fileModifiedAt')!,
updatedAt: mapValueOfType<String>(json, r'updatedAt')!,
@ -236,13 +219,12 @@ class AssetResponseDto {
isArchived: mapValueOfType<bool>(json, r'isArchived')!,
mimeType: mapValueOfType<String>(json, r'mimeType'),
duration: mapValueOfType<String>(json, r'duration')!,
webpPath: mapValueOfType<String>(json, r'webpPath'),
encodedVideoPath: mapValueOfType<String>(json, r'encodedVideoPath'),
exifInfo: ExifResponseDto.fromJson(json[r'exifInfo']),
smartInfo: SmartInfoResponseDto.fromJson(json[r'smartInfo']),
livePhotoVideoId: mapValueOfType<String>(json, r'livePhotoVideoId'),
tags: TagResponseDto.listFromJson(json[r'tags']),
people: PersonResponseDto.listFromJson(json[r'people']),
checksum: mapValueOfType<String>(json, r'checksum')!,
);
}
return null;
@ -297,7 +279,7 @@ class AssetResponseDto {
'deviceId',
'originalPath',
'originalFileName',
'resizePath',
'resized',
'fileCreatedAt',
'fileModifiedAt',
'updatedAt',
@ -305,7 +287,7 @@ class AssetResponseDto {
'isArchived',
'mimeType',
'duration',
'webpPath',
'checksum',
};
}

View file

@ -21,6 +21,7 @@ void main() {
// TODO
});
// base64 or hex encoded sha1 hash
// String checksum
test('to test the property `checksum`', () async {
// TODO

View file

@ -51,8 +51,8 @@ void main() {
// TODO
});
// String resizePath
test('to test the property `resizePath`', () async {
// bool resized
test('to test the property `resized`', () async {
// TODO
});
@ -91,16 +91,6 @@ void main() {
// TODO
});
// String webpPath
test('to test the property `webpPath`', () async {
// TODO
});
// String encodedVideoPath
test('to test the property `encodedVideoPath`', () async {
// TODO
});
// ExifResponseDto exifInfo
test('to test the property `exifInfo`', () async {
// TODO
@ -126,6 +116,12 @@ void main() {
// TODO
});
// base64 encoded sha1 hash
// String checksum
test('to test the property `checksum`', () async {
// TODO
});
});

View file

@ -30,6 +30,7 @@ import {
import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto';
import { BadRequestException, ForbiddenException } from '@nestjs/common';
import { when } from 'jest-when';
import { AssetRejectReason, AssetUploadAction } from './response-dto/asset-check-response.dto';
const _getCreateAssetDto = (): CreateAssetDto => {
const createAssetDto = new CreateAssetDto();
@ -504,4 +505,32 @@ describe('AssetService', () => {
expect(storageMock.createReadStream).toHaveBeenCalledWith('fake_path/asset_1.jpeg', 'image/jpeg');
});
});
describe('bulkUploadCheck', () => {
it('should accept hex and base64 checksums', async () => {
const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
const file2 = Buffer.from('53be335e99f18a66ff12e9a901c7a6171dd76573', 'hex');
assetRepositoryMock.getAssetsByChecksums.mockResolvedValue([
{ id: 'asset-1', checksum: file1 },
{ id: 'asset-2', checksum: file2 },
]);
await expect(
sut.bulkUploadCheck(authStub.admin, {
assets: [
{ id: '1', checksum: file1.toString('hex') },
{ id: '2', checksum: file2.toString('base64') },
],
}),
).resolves.toEqual({
results: [
{ id: '1', assetId: 'asset-1', action: AssetUploadAction.REJECT, reason: AssetRejectReason.DUPLICATE },
{ id: '2', assetId: 'asset-2', action: AssetUploadAction.REJECT, reason: AssetRejectReason.DUPLICATE },
],
});
expect(assetRepositoryMock.getAssetsByChecksums).toHaveBeenCalledWith(authStub.admin.id, [file1, file2]);
});
});
});

View file

@ -486,17 +486,24 @@ export class AssetService {
}
async bulkUploadCheck(authUser: AuthUserDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
// support base64 and hex checksums
for (const asset of dto.assets) {
if (asset.checksum.length === 28) {
asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
}
}
const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
const results = await this._assetRepository.getAssetsByChecksums(authUser.id, checksums);
const resultsMap: Record<string, string> = {};
const checksumMap: Record<string, string> = {};
for (const { id, checksum } of results) {
resultsMap[checksum.toString('hex')] = id;
checksumMap[checksum.toString('hex')] = id;
}
return {
results: dto.assets.map(({ id, checksum }) => {
const duplicate = resultsMap[checksum];
const duplicate = checksumMap[checksum];
if (duplicate) {
return {
id,

View file

@ -6,6 +6,7 @@ export class AssetBulkUploadCheckItem {
@IsNotEmpty()
id!: string;
/** base64 or hex encoded sha1 hash */
@IsString()
@IsNotEmpty()
checksum!: string;

View file

@ -4446,9 +4446,8 @@
"originalFileName": {
"type": "string"
},
"resizePath": {
"type": "string",
"nullable": true
"resized": {
"type": "boolean"
},
"fileCreatedAt": {
"type": "string"
@ -4472,14 +4471,6 @@
"duration": {
"type": "string"
},
"webpPath": {
"type": "string",
"nullable": true
},
"encodedVideoPath": {
"type": "string",
"nullable": true
},
"exifInfo": {
"$ref": "#/components/schemas/ExifResponseDto"
},
@ -4501,6 +4492,10 @@
"items": {
"$ref": "#/components/schemas/PersonResponseDto"
}
},
"checksum": {
"type": "string",
"description": "base64 encoded sha1 hash"
}
},
"required": [
@ -4511,7 +4506,7 @@
"deviceId",
"originalPath",
"originalFileName",
"resizePath",
"resized",
"fileCreatedAt",
"fileModifiedAt",
"updatedAt",
@ -4519,7 +4514,7 @@
"isArchived",
"mimeType",
"duration",
"webpPath"
"checksum"
]
},
"AlbumResponseDto": {
@ -6173,7 +6168,8 @@
"type": "string"
},
"checksum": {
"type": "string"
"type": "string",
"description": "base64 or hex encoded sha1 hash"
}
},
"required": [

View file

@ -1,6 +1,6 @@
import { AlbumEntity, AssetEntity, UserEntity } from '@app/infra/entities';
import { BadRequestException, ForbiddenException, Inject, Injectable } from '@nestjs/common';
import { IAssetRepository } from '../asset';
import { IAssetRepository, mapAsset } from '../asset';
import { AuthUserDto } from '../auth';
import { IJobRepository, JobName } from '../job';
import { IAlbumRepository } from './album.repository';
@ -40,6 +40,7 @@ export class AlbumService {
return albums.map((album) => {
return {
...album,
assets: album?.assets?.map(mapAsset),
sharedLinks: undefined, // Don't return shared links
shared: album.sharedLinks?.length > 0 || album.sharedUsers?.length > 0,
assetCount: albumsAssetCountObj[album.id],

View file

@ -15,7 +15,7 @@ export class AssetResponseDto {
type!: AssetType;
originalPath!: string;
originalFileName!: string;
resizePath!: string | null;
resized!: boolean;
fileCreatedAt!: string;
fileModifiedAt!: string;
updatedAt!: string;
@ -23,13 +23,13 @@ export class AssetResponseDto {
isArchived!: boolean;
mimeType!: string | null;
duration!: string;
webpPath!: string | null;
encodedVideoPath?: string | null;
exifInfo?: ExifResponseDto;
smartInfo?: SmartInfoResponseDto;
livePhotoVideoId?: string | null;
tags?: TagResponseDto[];
people?: PersonResponseDto[];
/**base64 encoded sha1 hash */
checksum!: string;
}
export function mapAsset(entity: AssetEntity): AssetResponseDto {
@ -41,21 +41,20 @@ export function mapAsset(entity: AssetEntity): AssetResponseDto {
type: entity.type,
originalPath: entity.originalPath,
originalFileName: entity.originalFileName,
resizePath: entity.resizePath,
resized: !!entity.resizePath,
fileCreatedAt: entity.fileCreatedAt,
fileModifiedAt: entity.fileModifiedAt,
updatedAt: entity.updatedAt,
isFavorite: entity.isFavorite,
isArchived: entity.isArchived,
mimeType: entity.mimeType,
webpPath: entity.webpPath,
encodedVideoPath: entity.encodedVideoPath,
duration: entity.duration ?? '0:00:00.00000',
exifInfo: entity.exifInfo ? mapExif(entity.exifInfo) : undefined,
smartInfo: entity.smartInfo ? mapSmartInfo(entity.smartInfo) : undefined,
livePhotoVideoId: entity.livePhotoVideoId,
tags: entity.tags?.map(mapTag),
people: entity.faces?.map(mapFace),
checksum: entity.checksum.toString('base64'),
};
}
@ -68,20 +67,19 @@ export function mapAssetWithoutExif(entity: AssetEntity): AssetResponseDto {
type: entity.type,
originalPath: entity.originalPath,
originalFileName: entity.originalFileName,
resizePath: entity.resizePath,
resized: !!entity.resizePath,
fileCreatedAt: entity.fileCreatedAt,
fileModifiedAt: entity.fileModifiedAt,
updatedAt: entity.updatedAt,
isFavorite: entity.isFavorite,
isArchived: entity.isArchived,
mimeType: entity.mimeType,
webpPath: entity.webpPath,
encodedVideoPath: entity.encodedVideoPath,
duration: entity.duration ?? '0:00:00.00000',
exifInfo: undefined,
smartInfo: entity.smartInfo ? mapSmartInfo(entity.smartInfo) : undefined,
livePhotoVideoId: entity.livePhotoVideoId,
tags: entity.tags?.map(mapTag),
people: entity.faces?.map(mapFace),
checksum: entity.checksum.toString('base64'),
};
}

View file

@ -3,7 +3,7 @@ import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common'
import { ConfigService } from '@nestjs/config';
import { mapAlbum } from '../album';
import { IAlbumRepository } from '../album/album.repository';
import { mapAsset } from '../asset';
import { AssetResponseDto, mapAsset } from '../asset';
import { IAssetRepository } from '../asset/asset.repository';
import { AuthUserDto } from '../auth';
import { MACHINE_LEARNING_ENABLED } from '../domain.constant';
@ -103,9 +103,13 @@ export class SearchService {
}
}
async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetEntity>[]> {
async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
this.assertEnabled();
return this.searchRepository.explore(authUser.id);
const results = await this.searchRepository.explore(authUser.id);
return results.map(({ fieldName, items }) => ({
fieldName,
items: items.map(({ value, data }) => ({ value, data: mapAsset(data) })),
}));
}
async search(authUser: AuthUserDto, dto: SearchDto): Promise<SearchResponseDto> {

View file

@ -446,7 +446,7 @@ const assetResponse: AssetResponseDto = {
type: AssetType.VIDEO,
originalPath: 'fake_path/jpeg',
originalFileName: 'asset_1.jpeg',
resizePath: '',
resized: false,
fileModifiedAt: today.toISOString(),
fileCreatedAt: today.toISOString(),
updatedAt: today.toISOString(),
@ -457,13 +457,12 @@ const assetResponse: AssetResponseDto = {
tags: [],
objects: ['a', 'b', 'c'],
},
webpPath: '',
encodedVideoPath: '',
duration: '0:00:00.00000',
exifInfo: assetInfo,
livePhotoVideoId: null,
tags: [],
people: [],
checksum: 'ZmlsZSBoYXNo',
};
const albumResponse: AlbumResponseDto = {

View file

@ -378,7 +378,7 @@ export interface AssetBulkUploadCheckItem {
*/
'id': string;
/**
*
* base64 or hex encoded sha1 hash
* @type {string}
* @memberof AssetBulkUploadCheckItem
*/
@ -586,10 +586,10 @@ export interface AssetResponseDto {
'originalFileName': string;
/**
*
* @type {string}
* @type {boolean}
* @memberof AssetResponseDto
*/
'resizePath': string | null;
'resized': boolean;
/**
*
* @type {string}
@ -632,18 +632,6 @@ export interface AssetResponseDto {
* @memberof AssetResponseDto
*/
'duration': string;
/**
*
* @type {string}
* @memberof AssetResponseDto
*/
'webpPath': string | null;
/**
*
* @type {string}
* @memberof AssetResponseDto
*/
'encodedVideoPath'?: string | null;
/**
*
* @type {ExifResponseDto}
@ -674,6 +662,12 @@ export interface AssetResponseDto {
* @memberof AssetResponseDto
*/
'people'?: Array<PersonResponseDto>;
/**
* base64 encoded sha1 hash
* @type {string}
* @memberof AssetResponseDto
*/
'checksum': string;
}

View file

@ -350,7 +350,7 @@
<div class="row-start-1 row-span-full col-start-1 col-span-4">
{#key asset.id}
{#if !asset.resizePath}
{#if !asset.resized}
<div class="h-full w-full flex justify-center">
<div
class="h-full bg-gray-100 dark:bg-immich-dark-gray flex items-center justify-center aspect-square px-auto"

View file

@ -123,7 +123,7 @@
</div>
{/if}
{#if asset.resizePath}
{#if asset.resized}
<ImageThumbnail
url={api.getAssetThumbnailUrl(asset.id, format, publicSharedKey)}
altText={asset.originalFileName}